@SuppressWarnings("unchecked")
	public <P> Populator<P> getPopulator(final Class<P> clazz, final Path path) {
		if (clazz == null || clazz.equals(Object.class)) {
			return (Populator<P>)NullPopulator.INSTANCE;
		} else {
			final ClassTranslator<P> classTranslator = (ClassTranslator<P>)this.<P, FullEntity<?>>getTranslator(new TypeKey<>(clazz), this, path);
			return classTranslator.getPopulator();
		}
	}

@ SuppressWarnings ( "unchecked" ) public < P > Populator < P > getPopulator ( final Class < P > clazz , final Path path ) { if ( clazz == null || clazz . equals ( Object . class ) ) { return ( Populator < P > ) NullPopulator . INSTANCE ; } else { final ClassTranslator < P > classTranslator = ( ClassTranslator < P > ) this . < P , FullEntity < ? > > getTranslator ( new TypeKey <> ( clazz ) , this , path ) ; return classTranslator . getPopulator ( ) ; } }

Get the populator for the specified class. This requires looking up the
translator for that class and then getting the populator from it.

@param clazz is the class we want a populator for.

Get the populator for the specified class . This requires looking up the translator for that class and then getting the populator from it .


private Boolean getIndexInstruction(Class<P> clazz) {
		Index ind = clazz.getAnnotation(Index.class);
		Unindex unind = clazz.getAnnotation(Unindex.class);

		if (ind != null && unind != null)
			throw new IllegalStateException("You cannot have @Index and @Unindex on the same class: " + clazz);

		if (ind != null)
			return true;
		else if (unind != null)
			return false;
		else
			return null;
	}

Figure out if there is an index instruction for the whole class.
@return true, false, or null (which means no info)

Figure out if there is an index instruction for the whole class .


private  boolean isOfInterest(Method method) {
		for (Annotation[] annos: method.getParameterAnnotations())
			if (TypeUtils.getAnnotation(annos, AlsoLoad.class) != null)
				return true;

		return false;
	}

private boolean isOfInterest ( Method method ) { for ( Annotation [ ] annos : method . getParameterAnnotations ( ) ) if ( TypeUtils . getAnnotation ( annos , AlsoLoad . class ) != null ) return true ; return false ; }

Determine if we should create a Property for the method (ie, @AlsoLoad)

Determine if we should create a Property for the method ( ie


private List<Property> getDeclaredProperties(ObjectifyFactory fact, Class<?> clazz) {
		List<Property> good = new ArrayList<>();

		for (Field field: clazz.getDeclaredFields())
			if (isOfInterest(field))
				good.add(new FieldProperty(fact, clazz, field));

		for (Method method: clazz.getDeclaredMethods())
			if (isOfInterest(method))
				good.add(new MethodProperty(method));

		return good;
	}

private List < Property > getDeclaredProperties ( ObjectifyFactory fact , Class < ? > clazz ) { List < Property > good = new ArrayList <> ( ) ; for ( Field field : clazz . getDeclaredFields ( ) ) if ( isOfInterest ( field ) ) good . add ( new FieldProperty ( fact , clazz , field ) ) ; for ( Method method : clazz . getDeclaredMethods ( ) ) if ( isOfInterest ( method ) ) good . add ( new MethodProperty ( method ) ) ; return good ; }

Get all the persistable fields and methods declared on a class. Ignores superclasses.

@return the fields we load and save, including @Id and @Parent fields. All fields will be set accessable
and returned in order of declaration.

Get all the persistable fields and methods declared on a class . Ignores superclasses .


public KeyMetadata<P> getKeyMetadata() {
		final Populator<Object> populator = props.get(0);
		Preconditions.checkState(populator instanceof KeyPopulator, "Cannot get KeyMetadata for non-@Entity class " + this.clazz);
		return ((KeyPopulator<P>)populator).getKeyMetadata();
	}

Gets the key metadata but only if this was an @Entity annotated class. Should not be called if not.


public <T> Result<T> get(final Key<T> key) {
		assert !isExecuted();

		SessionValue<T> sv = getSession().get(key);
		if (sv == null) {
			log.trace("Adding to round (session miss): {}", key);

			this.pending.add(key.getRaw());

			Result<T> result = new ResultCache<T>() {
				@Override
				@SuppressWarnings("unchecked")
				public T nowUncached() {
					// Because clients could conceivably get() in the middle of our operations (see LoadCollectionRefsTest.specialListWorks()),
					// we need to check for early execution. This will perform poorly, but at least it will work.
					//assert Round.this.isExecuted();
					loadEngine.execute();
					return (T)translated.now().get(key);
				}

				@Override
				public String toString() {
					return "(Fetch result for " + key + ")";
				}
			};

			sv = new SessionValue<>(result, getLoadArrangement());
			getSession().add(key, sv);

		} else {
			log.trace("Adding to round (session hit): {}", key);

			if (sv.loadWith(getLoadArrangement())) {
				log.trace("New load group arrangement, checking for upgrades: {}", getLoadArrangement());

				// We are looking at a brand-new arrangement for something that already existed in the session.
				// We need to go through any Ref<?>s that might be in need of loading. We find those refs by
				// actually saving the entity into a custom SaveContext.
				T thing = sv.getResult().now();
				if (thing != null) {
					SaveContext saveCtx = new SaveContext() {
						@Override
						public boolean skipLifecycle() {
							return true;
						}

						@Override
						public com.google.cloud.datastore.Key saveRef(Ref<?> value, LoadConditions loadConditions) {
							com.google.cloud.datastore.Key key = super.saveRef(value, loadConditions);

							if (loadEngine.shouldLoad(loadConditions)) {
								log.trace("Upgrading key {}", key);
								loadEngine.load(value.key());
							}

							return key;
						}
					};

					// We throw away the saved entity and we are done
					loadEngine.ofy.factory().getMetadataForEntity(thing).save(thing, saveCtx);
				}
			}
		}

		return sv.getResult();
	}

Gets a result, using the session cache if possible.

Gets a result using the session cache if possible .


public void execute() {
		if (needsExecution()) {
			log.trace("Executing round: {}", pending);

			Result<Map<com.google.cloud.datastore.Key, Entity>> fetched = fetchPending();
			translated = loadEngine.translate(fetched);

			// If we're in a transaction (and beyond the first round), force all subsequent rounds to complete.
			// This effectively means that only the first round can be asynchronous; all other rounds are
			// materialized immediately. The reason for this is that there are some nasty edge cases with @Load
			// annotations in transactions getting called after the transaction closes. This is possibly not the
			// best solution to the problem, but it solves the problem now.
			if (loadEngine.ofy.getTransaction() != null && depth > 0)
				translated.now();
		}
	}


private Result<Map<com.google.cloud.datastore.Key, Entity>> fetchPending() {
		// We don't need to fetch anything that has been stuffed

		final Map<com.google.cloud.datastore.Key, Entity> combined = new HashMap<>();
		Set<com.google.cloud.datastore.Key> fetch = new HashSet<>();

		for (com.google.cloud.datastore.Key key: pending) {
			Entity ent = stuffed.get(key);
			if (ent == null)
				fetch.add(key);
			else
				combined.put(key, ent);
		}

		if (fetch.isEmpty()) {
			return new ResultNow<>(combined);
		} else {
			final Result<Map<com.google.cloud.datastore.Key, Entity>> fetched = loadEngine.fetch(fetch);

			return () -> {
				combined.putAll(fetched.now());
				return combined;
			};
		}
	}

Possibly pulls some values from the stuffed collection


private void addIndexedDiscriminators(final Class<?> clazz) {
		if (clazz == Object.class)
			return;

		this.addIndexedDiscriminators(clazz.getSuperclass());

		final Subclass sub = clazz.getAnnotation(Subclass.class);
		if (sub != null && sub.index()) {
			final String disc = (sub.name().length() > 0) ? sub.name() : clazz.getSimpleName();
			this.indexedDiscriminators.add(StringValue.of(disc));
		}
	}

private void addIndexedDiscriminators ( final Class < ? > clazz ) { if ( clazz == Object . class ) return ; this . addIndexedDiscriminators ( clazz . getSuperclass ( ) ) ; final Subclass sub = clazz . getAnnotation ( Subclass . class ) ; if ( sub != null && sub . index ( ) ) { final String disc = ( sub . name ( ) . length ( ) > 0 ) ? sub . name ( ) : clazz . getSimpleName ( ) ; this . indexedDiscriminators . add ( StringValue . of ( disc ) ) ; } }

Recursively go through the class hierarchy adding any discriminators that are indexed


public void registerSubclass(ClassTranslator<? extends P> translator) {
		byDiscriminator.put(translator.getDiscriminator(), translator);

		Subclass sub = translator.getDeclaredClass().getAnnotation(Subclass.class);
		for (String alsoLoad: sub.alsoLoad())
			byDiscriminator.put(alsoLoad, translator);

		byClass.put(translator.getDeclaredClass(), translator);
	}

public void registerSubclass ( ClassTranslator < ? extends P > translator ) { byDiscriminator . put ( translator . getDiscriminator ( ) , translator ) ; Subclass sub = translator . getDeclaredClass ( ) . getAnnotation ( Subclass . class ) ; for ( String alsoLoad : sub . alsoLoad ( ) ) byDiscriminator . put ( alsoLoad , translator ) ; byClass . put ( translator . getDeclaredClass ( ) , translator ) ; }

Register a subclass translator with this class translator. That way if we get called upon
to translate an instance of the subclass, we will forward to the correct translator.

Register a subclass translator with this class translator . That way if we get called upon to translate an instance of the subclass we will forward to the correct translator .


public static Type getCollectionComponentType(Type collectionType) {
		Type componentType = GenericTypeReflector.getTypeParameter(collectionType, Collection.class.getTypeParameters()[0]);
		if (componentType == null)	// if it was a raw type, just assume Object
			return Object.class;
		else
			return componentType;
	}

Get the component type of a Collection .


public static Type getMapKeyType(Type mapType) {
		Type componentType = GenericTypeReflector.getTypeParameter(mapType, Map.class.getTypeParameters()[0]);
		if (componentType == null)	// if it was a raw type, just assume Object
			return Object.class;
		else
			return componentType;
	}


public static ForwardPath of(Path path) {
		ForwardPath next = new ForwardPath(path);
		if (path.getPrevious() == Path.root())
			return next;

		ForwardPath previous = of(path.getPrevious());
		previous.next = next;

		return previous;
	}

public static ForwardPath of ( Path path ) { ForwardPath next = new ForwardPath ( path ) ; if ( path . getPrevious ( ) == Path . root ( ) ) return next ; ForwardPath previous = of ( path . getPrevious ( ) ) ; previous . next = next ; return previous ; }

Recursive method which reverses the path into a ForwardPath.
@param path cannot be the root path.

Recursive method which reverses the path into a ForwardPath .


public Path getFinalPath() {
		ForwardPath here = this;
		while (here.next != null)
			here = here.next;

		return here.getPath();
	}

public Path getFinalPath ( ) { ForwardPath here = this ; while ( here . next != null ) here = here . next ; return here . getPath ( ) ; }

Get the complete path in this chain, typically for error messages or debugging

Get the complete path in this chain typically for error messages or debugging


public String toPathString() {
		if (this == ROOT) {
			return "";
		} else {
			StringBuilder builder = new StringBuilder();
			toPathString(builder);
			return builder.toString();
		}
	}

public String toPathString ( ) { if ( this == ROOT ) { return "" ; } else { StringBuilder builder = new StringBuilder ( ) ; toPathString ( builder ) ; return builder . toString ( ) ; } }


public int depth() {
		int depth = 0;

		Path here = this;
		while (here != ROOT) {
			depth++;
			here = here.previous;
		}

		return depth;
	}

public int depth ( ) { int depth = 0 ; Path here = this ; while ( here != ROOT ) { depth ++ ; here = here . previous ; } return depth ; }

ROOT is 0, top level Entity properties are 1, embedded things are higher.

ROOT is 0 top level Entity properties are 1 embedded things are higher .


public void putAll(final Collection<Bucket> updates) {
		final Set<Key> good = this.cachePutIfUntouched(updates);

		if (good.size() == updates.size())
			return;

		// Figure out which ones were bad
		final List<Key> bad = updates.stream()
				.map(Bucket::getKey)
				.filter(key -> !good.contains(key))
				.collect(Collectors.toList());

		if (!bad.isEmpty()) {
			// So we had some collisions.  We need to reset these back to null, but do it in a safe way - if we
			// blindly set null something already null, it will break any putIfUntouched() which saw the first null.
			// This could result in write contention starving out a real write.  The solution is to only reset things
			// that are not already null.

			final Map<Key, Object> cached = this.cacheGetAll(bad);

			// Remove the stuff we don't care about
			cached.values().removeIf(Objects::isNull);

			this.empty(cached.keySet());
		}
	}

Update a set of buckets with new values.  If collisions occur, resets the memcache value to null.

@param updates can have null Entity values, which will record a negative cache result.  Buckets must have
been obtained from getAll().

Update a set of buckets with new values . If collisions occur resets the memcache value to null .


public void empty(final Iterable<Key> keys) {
		final Map<Key, Object> updates = new HashMap<>();

		for (final Key key: keys)
			if (cacheControl.isCacheable(key))
				updates.put(key, null);

		this.memcacheWithRetry.putAll(updates);
	}

public void empty ( final Iterable < Key > keys ) { final Map < Key , Object > updates = new HashMap <> ( ) ; for ( final Key key : keys ) if ( cacheControl . isCacheable ( key ) ) updates . put ( key , null ) ; this . memcacheWithRetry . putAll ( updates ) ; }

Revert a set of keys to the empty state.  Will loop on this several times just in case
the memcache write fails - we don't want to leave the cache in a nasty state.

Revert a set of keys to the empty state . Will loop on this several times just in case the memcache write fails - we don t want to leave the cache in a nasty state .


private Set<Key> cachePutIfUntouched(final Iterable<Bucket> buckets) {
		final Map<Key, CasPut> payload = new HashMap<>();
		final Set<Key> successes = new HashSet<>();

		for (final Bucket buck: buckets) {
			if (!buck.isCacheable()) {
				successes.add(buck.getKey());
				continue;
			}

			final Integer expirySeconds = cacheControl.getExpirySeconds(buck.getKey());
			if (expirySeconds == null) {
				successes.add(buck.getKey());
				continue;
			}

			payload.put(buck.getKey(), new CasPut(buck.identifiableValue, buck.getNextToStore(), expirySeconds));
		}

		successes.addAll(this.memcache.putIfUntouched(payload));

		return successes;
	}

private Set < Key > cachePutIfUntouched ( final Iterable < Bucket > buckets ) { final Map < Key , CasPut > payload = new HashMap <> ( ) ; final Set < Key > successes = new HashSet <> ( ) ; for ( final Bucket buck : buckets ) { if ( ! buck . isCacheable ( ) ) { successes . add ( buck . getKey ( ) ) ; continue ; } final Integer expirySeconds = cacheControl . getExpirySeconds ( buck . getKey ( ) ) ; if ( expirySeconds == null ) { successes . add ( buck . getKey ( ) ) ; continue ; } payload . put ( buck . getKey ( ) , new CasPut ( buck . identifiableValue , buck . getNextToStore ( ) , expirySeconds ) ) ; } successes . addAll ( this . memcache . putIfUntouched ( payload ) ) ; return successes ; }

Put buckets in the cache, checking for cacheability and collisions.
@return the set of keys that were *successfully* handled. That includes buckets that were put without collision
and buckets that didn't need to be cached.

Put buckets in the cache checking for cacheability and collisions .


private Map<Key, Object> cacheGetAll(final Collection<Key> keys) {
		try {
			return this.memcache.getAll(keys);
		} catch (Exception ex) {
			// Some sort of serialization error, just wipe out the values
			log.warn("Error fetching values from memcache, deleting keys", ex);

			this.memcache.deleteAll(keys);

			return new HashMap<>();
		}
	}

Bulk get on keys, getting the raw objects

Bulk get on keys getting the raw objects


public static Set<Key> keysOf(final Collection<Bucket> buckets) {
		return buckets.stream().map(Bucket::getKey).collect(Collectors.toSet());
	}

public static Set < Key > keysOf ( final Collection < Bucket > buckets ) { return buckets . stream ( ) . map ( Bucket :: getKey ) . collect ( Collectors . toSet ( ) ) ; }

Basically a list comprehension of the keys for convenience.

Basically a list comprehension of the keys for convenience .


private void findKeyFields(Class<?> inspect, CreateContext ctx, Path path) {
		if (inspect == Object.class)
			return;

		findKeyFields(inspect.getSuperclass(), ctx, path);

		for (Field field: inspect.getDeclaredFields()) {
			if (field.getAnnotation(Id.class) != null) {
				if (this.idMeta != null)
					throw new IllegalStateException("Multiple @Id fields in the class hierarchy of " + clazz.getName());

				if ((field.getType() != Long.class) && (field.getType() != long.class) && (field.getType() != String.class))
					throw new IllegalStateException("@Id field '" + field.getName() + "' in " + inspect.getName() + " must be of type Long, long, or String");

				Property prop = new FieldProperty(ctx.getFactory(), clazz, field);
				Translator<Object, Object> translator = ctx.getTranslator(new TypeKey<>(prop), ctx, path.extend(prop.getName()));

				this.idMeta = new PropertyPopulator<>(prop, translator);

			} else if (field.getAnnotation(Parent.class) != null) {
				if (this.parentMeta != null)
					throw new IllegalStateException("Multiple @Parent fields in the class hierarchy of " + clazz.getName());

				if (!isAllowedParentFieldType(field.getType()))
					throw new IllegalStateException("@Parent fields must be Ref<?>, Key<?>, or datastore Key. Illegal parent: " + field);

				Property prop = new FieldProperty(ctx.getFactory(), clazz, field);
				Translator<Object, Object> translator = ctx.getTranslator(new TypeKey<>(prop), ctx, path.extend(prop.getName()));

				this.parentMeta = new PropertyPopulator<>(prop, translator);
			}
		}
	}

Recursively go through the class hierarchy looking for the idMeta and parentMeta fields.
@throws IllegalStateException if @Id or @Parent shows up twice.

Recursively go through the class hierarchy looking for the idMeta and parentMeta fields .


@SuppressWarnings("unchecked")
	public <K extends IncompleteKey> void setKey(final FullEntity.Builder<K> container, final P pojo) {
		final IncompleteKey rawKey = getIncompleteKey(pojo);

		if (!(rawKey instanceof com.google.cloud.datastore.Key)) {
			// it's incomplete, make sure we can save it
			Preconditions.checkState(isIdNumeric(), "Cannot save an entity with a null String @Id: %s", pojo);
		}

		container.setKey((K)rawKey);
	}

Sets the key on a container from the POJO.

Sets the key on a container from the POJO .


public void setLongId(P pojo, Long id) {
		if (!clazz.isAssignableFrom(pojo.getClass()))
			throw new IllegalArgumentException("Trying to use metadata for " + clazz.getName() + " to set key of " + pojo.getClass().getName());

		this.idMeta.getProperty().set(pojo, id);
	}

public
void
setLongId
(
P
pojo
,
Long
id
)
{
if
(
!
clazz
.
isAssignableFrom
(
pojo
.
getClass
(
)
)
)
throw
new
IllegalArgumentException
(
"Trying to use metadata for "
+
clazz
.
getName
(
)
+
" to set key of "
+
pojo
.
getClass
(
)
.
getName
(
)
)
;
this
.
idMeta
.
getProperty
(
)
.
set
(
pojo
,
id
)
;
}


private com.google.cloud.datastore.Key getParentRaw(P pojo) {
		if (parentMeta == null)
			return null;

		final Value<Object> value = parentMeta.getValue(pojo, new SaveContext(), Path.root());
		return (value == null || value.getType() == ValueType.NULL)
				? null
				: (com.google.cloud.datastore.Key)value.get();
	}

private com . google . cloud . datastore . Key getParentRaw ( P pojo ) { if ( parentMeta == null ) return null ; final Value < Object > value = parentMeta . getValue ( pojo , new SaveContext ( ) , Path . root ( ) ) ; return ( value == null || value . getType ( ) == ValueType . NULL ) ? null : ( com . google . cloud . datastore . Key ) value . get ( ) ; }

Get the contents of the @Parent field as a datastore key.
@return null if there was no @Parent field, or the field is null.


private void registerSubclass(final ClassTranslator<P> translator, final TypeKey<? super P> superclassTypeKey, final CreateContext ctx, final Path path) {
		if (superclassTypeKey.getTypeAsClass() == Object.class)
			return;

		@SuppressWarnings("unchecked")
		final ClassTranslator<? super P> superTranslator = create((TypeKey)superclassTypeKey, ctx, path);
		superTranslator.registerSubclass(translator);

		registerSubclass(translator, new TypeKey<>(superclassTypeKey.getTypeAsClass().getSuperclass()), ctx, path);
	}

private void registerSubclass ( final ClassTranslator < P > translator , final TypeKey < ? super P > superclassTypeKey , final CreateContext ctx , final Path path ) { if ( superclassTypeKey . getTypeAsClass ( ) == Object . class ) return ; @ SuppressWarnings ( "unchecked" ) final ClassTranslator < ? super P > superTranslator = create ( ( TypeKey ) superclassTypeKey , ctx , path ) ; superTranslator . registerSubclass ( translator ) ; registerSubclass ( translator , new TypeKey <> ( superclassTypeKey . getTypeAsClass ( ) . getSuperclass ( ) ) , ctx , path ) ; }

Recursively register this subclass with all the superclass translators. This works because we cache
translators uniquely in the factory.

Recursively register this subclass with all the superclass translators . This works because we cache translators uniquely in the factory .


private static boolean couldHaveCommonSubtype(Type type1, Type type2) {
		// this is an optimistically naive implementation.
		// if they are parameterized types their parameters need to be checked,...
		// so we're just a bit too lenient here

		Class<?> erased1 = GenericTypeReflector.erase(type1);
		Class<?> erased2 = GenericTypeReflector.erase(type2);
		// if they are both classes
		if (!erased1.isInterface() &&  !erased2.isInterface()) {
			// then one needs to be a subclass of another
			if (!erased1.isAssignableFrom(erased2) && !erased2.isAssignableFrom(erased1)) {
				return false;
			}
		}
		return true;
	}

Checks if the intersection of two types is not empty.

Checks if the intersection of two types is not empty .


private static Type transformOwner(Type givenOwner, Class<?> clazz) {
		if (givenOwner == null) {
			// be lenient: if this is an inner class but no owner was specified, assume a raw owner type
			// (or if there is no owner just return null)
			return clazz.getDeclaringClass();
		} else {
			// If the specified owner is not of the declaring class' type, but instead a subtype,
			// transform it into the declaring class with the exact type parameters.
			// For example with "class StringOuter extends GenericOuter<String>", transform
			// "StringOuter.Inner" into "GenericOuter<String>.Inner", just like the Java compiler does.
			Type transformedOwner = GenericTypeReflector.getExactSuperType(givenOwner, clazz.getDeclaringClass());

			if (transformedOwner == null) { // null means it's not a supertype
				throw new IllegalArgumentException("Given owner type [" + givenOwner + "] is not appropriate for ["
						+ clazz + "]: it should be a subtype of " + clazz.getDeclaringClass());
			}

			if (Modifier.isStatic(clazz.getModifiers())) {
				// for a static inner class, the owner shouldn't have type parameters
				return GenericTypeReflector.erase(transformedOwner);
			} else {
				return transformedOwner;
			}
		}
	}

Transforms the given owner type into an appropriate one when constructing a parameterized type.

Transforms the given owner type into an appropriate one when constructing a parameterized type .


public <A extends Annotation> A getAnnotationAnywhere(Class<A> annotationType) {
		A anno = getAnnotation(annotationType);
		if (anno == null) {
			Class<?> clazz = (Class<?>) GenericTypeReflector.erase(type);
			return clazz.getAnnotation(annotationType);
		} else {
			return anno;
		}
	}

public < A extends Annotation > A getAnnotationAnywhere ( Class < A > annotationType ) { A anno = getAnnotation ( annotationType ) ; if ( anno == null ) { Class < ? > clazz = ( Class < ? > ) GenericTypeReflector . erase ( type ) ; return clazz . getAnnotation ( annotationType ) ; } else { return anno ; } }

Checks not only the listed annotations but also annotations on the class.

Checks not only the listed annotations but also annotations on the class .


public void addAll(final Session other) {
		if (log.isTraceEnabled())
			log.trace("Adding all values to session: {}", other.map.keySet());

		map.putAll(other.map);
	}

public
void
addAll
(
final
Session
other
)
{
if
(
log
.
isTraceEnabled
(
)
)
log
.
trace
(
"Adding all values to session: {}"
,
other
.
map
.
keySet
(
)
)
;
map
.
putAll
(
other
.
map
)
;
}

Add all entries in the other session to this one


public <T> Result<T> load(final Key<T> key) {
		if (key == null)
			throw new NullPointerException("You tried to load a null key!");

		final Result<T> result = round.get(key);

		// If we are running a transaction, enlist the result so that it gets processed on commit even
		// if the client never materializes the result.
		if (ofy.getTransaction() != null)
			((PrivateAsyncTransaction)ofy.getTransaction()).enlist(result);

		// Now check to see if we need to recurse and add our parent(s) to the round
		if (key.getParent() != null) {
			final KeyMetadata<?> meta = ofy.factory().keys().getMetadata(key);
			// Is it really possible for this to be null?
			if (meta != null) {
				if (meta.shouldLoadParent(loadArrangement)) {
					load(key.getParent());
				}
			}
		}

		return result;
	}

Gets the result, possibly from the session, putting it in the session if necessary.
Also will recursively prepare the session with @Load parents as appropriate.
@throws NullPointerException if key is null

Gets the result possibly from the session putting it in the session if necessary . Also will recursively prepare the session with


public void execute() {
		if (round.needsExecution()) {
			Round old = round;
			round = old.next();
			old.execute();
		}
	}

public void execute ( ) { if ( round . needsExecution ( ) ) { Round old = round ; round = old . next ( ) ; old . execute ( ) ; } }

Starts asychronous fetching of the batch.

Starts asychronous fetching of the batch .


public Result<Map<Key<?>, Object>> translate(final Result<Map<com.google.cloud.datastore.Key, Entity>> raw) {
		return new ResultCache<Map<Key<?>, Object>>() {

			/** */
			private LoadContext ctx;

			/** */
			@Override
			public Map<Key<?>, Object> nowUncached() {
				final Map<Key<?>, Object> result = new HashMap<>(raw.now().size() * 2);

				ctx = new LoadContext(LoadEngine.this);

				for (final Entity ent: raw.now().values()) {
					final Key<?> key = Key.create(ent.getKey());
					final Object entity = load(ent, ctx);
					result.put(key, entity);
				}

				return result;
			}

			/**
			 * We need to execute the done() after the translated value has been set, otherwise we
			 * can produce an infinite recursion problem.
			 */
			@Override
			protected void postExecuteHook() {
				ctx.done();
				ctx = null;
			}
		};
	}

Asynchronously translate raw to processed; might produce successive load operations as refs are filled in

Asynchronously translate raw to processed ; might produce successive load operations as refs are filled in


public Result<Map<com.google.cloud.datastore.Key, Entity>> fetch(Set<com.google.cloud.datastore.Key> keys) {
		log.debug("Fetching {} keys: {}", keys.size(), keys);

		final Future<Map<com.google.cloud.datastore.Key, Entity>> fut = datastore.get(keys, readOptions.toArray(new ReadOption[readOptions.size()]));
		return ResultAdapter.create(fut);
	}

public
Result
<
Map
<
com
.
google
.
cloud
.
datastore
.
Key
,
Entity
>
>
fetch
(
Set
<
com
.
google
.
cloud
.
datastore
.
Key
>
keys
)
{
log
.
debug
(
"Fetching {} keys: {}"
,
keys
.
size
(
)
,
keys
)
;
final
Future
<
Map
<
com
.
google
.
cloud
.
datastore
.
Key
,
Entity
>
>
fut
=
datastore
.
get
(
keys
,
readOptions
.
toArray
(
new
ReadOption
[
readOptions
.
size
(
)
]
)
)
;
return
ResultAdapter
.
create
(
fut
)
;
}

Fetch the keys from the async datastore using the current transaction context


@SuppressWarnings("unchecked")
	public <T> T load(final BaseEntity<com.google.cloud.datastore.Key> ent, final LoadContext ctx) {
		if (ent == null)
			return null;

		final EntityMetadata<T> meta = ofy.factory().getMetadata(ent.getKey().getKind());
		if (meta == null)
			return (T)ent;
		else
			return meta.load(ent, ctx);
	}

@ SuppressWarnings ( "unchecked" ) public < T > T load ( final BaseEntity < com . google . cloud . datastore . Key > ent , final LoadContext ctx ) { if ( ent == null ) return null ; final EntityMetadata < T > meta = ofy . factory ( ) . getMetadata ( ent . getKey ( ) . getKind ( ) ) ; if ( meta == null ) return ( T ) ent ; else return meta . load ( ent , ctx ) ; }

Converts a datastore entity into a typed pojo object
@return an assembled pojo, or the Entity itself if the kind is not registered, or null if the input value was null


public com.google.cloud.datastore.Key createRawAny(final com.google.cloud.datastore.Key parent, final String kind, final Object id) {
		if (id instanceof String)
			return createRaw(parent, kind, (String)id);
		else if (id instanceof Long)
			return createRaw(parent, kind, (Long)id);
		else
			throw new IllegalArgumentException("id '" + id + "' must be String or Long");
	}

public
com
.
google
.
cloud
.
datastore
.
Key
createRawAny
(
final
com
.
google
.
cloud
.
datastore
.
Key
parent
,
final
String
kind
,
final
Object
id
)
{
if
(
id
instanceof
String
)
return
createRaw
(
parent
,
kind
,
(
String
)
id
)
;
else
if
(
id
instanceof
Long
)
return
createRaw
(
parent
,
kind
,
(
Long
)
id
)
;
else
throw
new
IllegalArgumentException
(
"id '"
+
id
+
"' must be String or Long"
)
;
}

Construct a Key from a Long or String id
@param id must be either Long or String


public static com.google.cloud.datastore.Key raw(final Key<?> key) {
		return key == null ? null : key.getRaw();
	}

public static com . google . cloud . datastore . Key raw ( final Key < ? > key ) { return key == null ? null : key . getRaw ( ) ; }


@SuppressWarnings("unchecked")
	public static <S> Value<S> getIdValue(final IncompleteKey key) {
		if (key instanceof com.google.cloud.datastore.Key) {
			final com.google.cloud.datastore.Key completeKey = (com.google.cloud.datastore.Key)key;
			if (completeKey.hasId())
				return (Value<S>)LongValue.of(completeKey.getId());
			else
				return (Value<S>)StringValue.of(completeKey.getName());
		} else {
			return null;
		}
	}

@ SuppressWarnings ( "unchecked" ) public static < S > Value < S > getIdValue ( final IncompleteKey key ) { if ( key instanceof com . google . cloud . datastore . Key ) { final com . google . cloud . datastore . Key completeKey = ( com . google . cloud . datastore . Key ) key ; if ( completeKey . hasId ( ) ) return ( Value < S > ) LongValue . of ( completeKey . getId ( ) ) ; else return ( Value < S > ) StringValue . of ( completeKey . getName ( ) ) ; } else { return null ; } }

Gets the String or Long id from the key as a Value, or null if incomplete

Gets the String or Long id from the key as a Value or null if incomplete


@SneakyThrows
	public static com.google.cloud.datastore.Key fromUrlSafe(final String urlSafeKey) {
		if (urlSafeKey.startsWith("a")) {
			return KeyFormat.INSTANCE.parseOldStyleAppEngineKey(urlSafeKey);
		} else {
			return com.google.cloud.datastore.Key.fromUrlSafe(urlSafeKey);
		}
	}

@ SneakyThrows public static com . google . cloud . datastore . Key fromUrlSafe ( final String urlSafeKey ) { if ( urlSafeKey . startsWith ( "a" ) ) { return KeyFormat . INSTANCE . parseOldStyleAppEngineKey ( urlSafeKey ) ; } else { return com . google . cloud . datastore . Key . fromUrlSafe ( urlSafeKey ) ; } }

Understands both the legacy format "ag1zfnZvb2Rvb2R5bmUwcgcLEgFCGAEM" and new format,
providing the key either way.

Understands both the legacy format ag1zfnZvb2Rvb2R5bmUwcgcLEgFCGAEM and new format providing the key either way .


@Override
	public ObjectifyImpl transactionless(final ObjectifyImpl parent) {
		return parent.makeNew(next -> new TransactorNo(next, parentTransactor.getSession()));
	}

@ Override public ObjectifyImpl transactionless ( final ObjectifyImpl parent ) { return parent . makeNew ( next -> new TransactorNo ( next , parentTransactor . getSession ( ) ) ) ; }

This version goes back to life without a transaction, but preserves current options.
We use the session from the parent, ie life before transactions.

This version goes back to life without a transaction but preserves current options . We use the session from the parent ie life before transactions .


@Override
	public <R> R transactNew(final ObjectifyImpl parent, final int limitTries, final Work<R> work) {
		return transactionless(parent).transactNew(limitTries, work);
	}

@ Override public < R > R transactNew ( final ObjectifyImpl parent , final int limitTries , final Work < R > work ) { return transactionless ( parent ) . transactNew ( limitTries , work ) ; }

We need to make sure the parentSession is the transactionless session, not the session
for our transaction.  This gives proper transaction isolation.

We need to make sure the parentSession is the transactionless session not the session for our transaction . This gives proper transaction isolation .


private <R> R transactOnce(final ObjectifyImpl parent, final Work<R> work) {
		final ObjectifyImpl txnOfy = factory.open(parent.getOptions(), next -> new TransactorYes(next, this));

		boolean committedSuccessfully = false;
		try {
			final R result = work.run();
			txnOfy.flush();
			txnOfy.getTransaction().commit();
			committedSuccessfully = true;
			return result;
		}
		finally {
			if (txnOfy.getTransaction().isActive()) {
				try {
					txnOfy.getTransaction().rollback();
				} catch (RuntimeException ex) {
					log.error("Rollback failed, suppressing error", ex);
				}
			}

			txnOfy.close();

			if (committedSuccessfully) {
				((PrivateAsyncTransaction)txnOfy.getTransaction()).runCommitListeners();
			}
		}
	}

private
<
R
>
R
transactOnce
(
final
ObjectifyImpl
parent
,
final
Work
<
R
>
work
)
{
final
ObjectifyImpl
txnOfy
=
factory
.
open
(
parent
.
getOptions
(
)
,
next
->
new
TransactorYes
(
next
,
this
)
)
;
boolean
committedSuccessfully
=
false
;
try
{
final
R
result
=
work
.
run
(
)
;
txnOfy
.
flush
(
)
;
txnOfy
.
getTransaction
(
)
.
commit
(
)
;
committedSuccessfully
=
true
;
return
result
;
}
finally
{
if
(
txnOfy
.
getTransaction
(
)
.
isActive
(
)
)
{
try
{
txnOfy
.
getTransaction
(
)
.
rollback
(
)
;
}
catch
(
RuntimeException
ex
)
{
log
.
error
(
"Rollback failed, suppressing error"
,
ex
)
;
}
}
txnOfy
.
close
(
)
;
if
(
committedSuccessfully
)
{
(
(
PrivateAsyncTransaction
)
txnOfy
.
getTransaction
(
)
)
.
runCommitListeners
(
)
;
}
}
}


private boolean matches(Object onPojo, If<?, ?>[] conditions) {
		if (conditions == null)
			return false;

		Object value = this.get(onPojo);

		for (If<?, ?> condition: conditions) {
			@SuppressWarnings("unchecked")
			If<Object, Object> cond = (If<Object, Object>)condition;

			if (cond.matchesValue(value))
				return true;

			if (cond.matchesPojo(onPojo))
				return true;
		}

		return false;
	}

private boolean matches ( Object onPojo , If < ? , ? > [ ] conditions ) { if ( conditions == null ) return false ; Object value = this . get ( onPojo ) ; for ( If < ? , ? > condition : conditions ) { @ SuppressWarnings ( "unchecked" ) If < Object , Object > cond = ( If < Object , Object > ) condition ; if ( cond . matchesValue ( value ) ) return true ; if ( cond . matchesPojo ( onPojo ) ) return true ; } return false ; }

Tests whether a set of conditions match.
@param conditions can be null; this always matches false
@return true if we match the conditions, false if we do not

Tests whether a set of conditions match .


private Objectify ofy() {
		// If we have an expired transaction context, we need a new context
		if (ofy == null || (ofy.getTransaction() != null && !ofy.getTransaction().isActive()))
			ofy = ObjectifyService.ofy();

		return ofy;
	}

Get the current objectify instance associated with this ref


public AsyncDatastore asyncDatastore(final boolean enableGlobalCache) {
		if (this.entityMemcache != null && enableGlobalCache && this.registrar.isCacheEnabled())
			return new CachingAsyncDatastore(asyncDatastore(), this.entityMemcache);
		else
			return asyncDatastore();
	}

public AsyncDatastore asyncDatastore ( final boolean enableGlobalCache ) { if ( this . entityMemcache != null && enableGlobalCache && this . registrar . isCacheEnabled ( ) ) return new CachingAsyncDatastore ( asyncDatastore ( ) , this . entityMemcache ) ; else return asyncDatastore ( ) ; }

Might produce a caching version if caching is enabled.

Might produce a caching version if caching is enabled .


@SuppressWarnings("unchecked")
	public <T> EntityMetadata<T> getMetadataForEntity(final T obj) throws IllegalArgumentException {
		// Type erasure sucks
		return (EntityMetadata<T>)this.getMetadata(obj.getClass());
	}

@
SuppressWarnings
(
"unchecked"
)
public
<
T
>
EntityMetadata
<
T
>
getMetadataForEntity
(
final
T
obj
)
throws
IllegalArgumentException
{
// Type erasure sucks
return
(
EntityMetadata
<
T
>
)
this
.
getMetadata
(
obj
.
getClass
(
)
)
;
}

Named differently so you don't accidentally use the Object form
@return the metadata for a kind of typed object.
@throws IllegalArgumentException if the kind has not been registered

Named differently so you don t accidentally use the Object form


private <T> KeyRange<T> allocate(final IncompleteKey incompleteKey, final int num) {
		final IncompleteKey[] allocations = new IncompleteKey[num];
		Arrays.fill(allocations, incompleteKey);

		final List<Key<T>> typedKeys = datastore().allocateId(allocations).stream()
				.map(Key::<T>create)
				.collect(Collectors.toList());

		return new KeyRange<>(typedKeys);
	}

private < T > KeyRange < T > allocate ( final IncompleteKey incompleteKey , final int num ) { final IncompleteKey [ ] allocations = new IncompleteKey [ num ] ; Arrays . fill ( allocations , incompleteKey ) ; final List < Key < T > > typedKeys = datastore ( ) . allocateId ( allocations ) . stream ( ) . map ( Key :: < T > create ) . collect ( Collectors . toList ( ) ) ; return new KeyRange <> ( typedKeys ) ; }

Allocate num copies of the incompleteKey


public ObjectifyImpl open(final ObjectifyOptions opts, final TransactorSupplier transactorSupplier) {
		final ObjectifyImpl objectify = new ObjectifyImpl(this, opts, transactorSupplier);
		stacks.get().add(objectify);
		return objectify;
	}

public ObjectifyImpl open ( final ObjectifyOptions opts , final TransactorSupplier transactorSupplier ) { final ObjectifyImpl objectify = new ObjectifyImpl ( this , opts , transactorSupplier ) ; stacks . get ( ) . add ( objectify ) ; return objectify ; }

This is only public because it is used from the impl package; don't use this as a public API

This is only public because it is used from the impl package ; don t use this as a public API


public void close(final Objectify ofy) {
		final Deque<Objectify> stack = stacks.get();
		if (stack.isEmpty())
			throw new IllegalStateException("You have already destroyed the Objectify context.");

		final Objectify popped = stack.removeLast();
		assert popped == ofy : "Mismatched objectify instances; somehow the stack was corrupted";
	}

Pops context off of stack after a transaction completes. For internal housekeeping only.

Pops context off of stack after a transaction completes . For internal housekeeping only .


public static void homogenizeIndexes(final List<Value<?>> list) {
		if (isIndexHomogeneous(list))
			return;

		for (int i=0; i<list.size(); i++) {
			final Value<?> value = list.get(i);
			if (value.excludeFromIndexes())
				list.set(i, index(value, true));
		}
	}

public static void homogenizeIndexes ( final List < Value < ? > > list ) { if ( isIndexHomogeneous ( list ) ) return ; for ( int i = 0 ; i < list . size ( ) ; i ++ ) { final Value < ? > value = list . get ( i ) ; if ( value . excludeFromIndexes ( ) ) list . set ( i , index ( value , true ) ) ; } }

The datastore has a weird behavior of reordering values in a list so that indexed ones come before nonindexed
ones. This can really mess up ordered lists. So if we find a heterogeneous list, we need to force index everything.

The datastore has a weird behavior of reordering values in a list so that indexed ones come before nonindexed ones . This can really mess up ordered lists . So if we find a heterogeneous list we need to force index everything .


public static <T> Ref<T> create(T value) {
		Key<T> key = Key.create(value);
		return create(key);
	}

public static < T > Ref < T > create ( T value ) { Key < T > key = Key . create ( value ) ; return create ( key ) ; }

Creates a Ref from a registered pojo entity


final public T safe() throws NotFoundException {
		T t = this.get();
		if (t == null)
			throw new NotFoundException(key());
		else
			return t;
	}

final public T safe ( ) throws NotFoundException { T t = this . get ( ) ; if ( t == null ) throw new NotFoundException ( key ( ) ) ; else return t ; }

Obtain the entity value, throwing an exception if the entity was not found.

@return the entity referenced. Never returns null.
@throws NotFoundException if the specified entity was not found

Obtain the entity value throwing an exception if the entity was not found .


public boolean add(K key, V value) {

		List<V> list = this.get(key);
		if (list == null) {
			list = new ArrayList<>();
			this.put(key, list);
		}

		return list.add(value);
	}

public boolean add ( K key , V value ) { List < V > list = this . get ( key ) ; if ( list == null ) { list = new ArrayList <> ( ) ; this . put ( key , list ) ; } return list . add ( value ) ; }

Adds a value to the set associated with the key.

Adds a value to the set associated with the key .


@Override
	public boolean isDone() {
		boolean done = this.raw.isDone();

		if (!triggered && done) {
			this.triggered = true;
			PendingFutures.removePending(this);

			this.trigger();
		}

		return done;
	}

@ Override public boolean isDone ( ) { boolean done = this . raw . isDone ( ) ; if ( ! triggered && done ) { this . triggered = true ; PendingFutures . removePending ( this ) ; this . trigger ( ) ; } return done ; }

This version also checks to see if we are done and we still need to call the trigger.
If so, it calls it.

@see java.util.concurrent.Future#isDone()

This version also checks to see if we are done and we still need to call the trigger . If so it calls it .


public If<?, ?>[] generateIfConditions(Class<? extends If<?, ?>>[] ifClasses, Field field) {
		if (ifClasses.length == 0)
			return ALWAYS;

		If<?, ?>[] result = new If<?, ?>[ifClasses.length];

		for (int i=0; i<ifClasses.length; i++) {

			Class<? extends If<?, ?>> ifClass = ifClasses[i];
			result[i] = this.createIf(ifClass, field);

			// Sanity check the generic If class types to ensure that they match the actual types of the field & entity.

			Type valueType = GenericTypeReflector.getTypeParameter(ifClass, If.class.getTypeParameters()[0]);
			Class<?> valueClass = GenericTypeReflector.erase(valueType);

			Type pojoType = GenericTypeReflector.getTypeParameter(ifClass, If.class.getTypeParameters()[1]);
			Class<?> pojoClass = GenericTypeReflector.erase(pojoType);

			if (!TypeUtils.isAssignableFrom(valueClass, field.getType()))
				throw new IllegalStateException("Cannot use If class " + ifClass.getName() + " on " + field
						+ " because you cannot assign " + field.getType().getName() + " to " + valueClass.getName());

			if (!TypeUtils.isAssignableFrom(pojoClass, field.getDeclaringClass()))
				throw new IllegalStateException("Cannot use If class " + ifClass.getName() + " on " + field
						+ " because the containing class " + field.getDeclaringClass().getName() + " is not compatible with " + pojoClass.getName());
		}

		return result;
	}

Clever enough to recognize that an empty set of conditions means Always.

Clever enough to recognize that an empty set of conditions means Always .


public void done() {
		engine.execute();

		while (deferred != null) {
			final List<Runnable> runme = deferred;
			deferred = null;	// reset this because it might get filled with more

			for (final Runnable run: runme) {
				log.trace("Executing {}", run);
				run.run();
			}
		}
	}

Call this when a load process completes.  Executes anything in the batch and then executes any delayed operations.

Call this when a load process completes . Executes anything in the batch and then executes any delayed operations .


public Object getContainer(Type containerType, Path path) {
		Class<?> containerClass = GenericTypeReflector.erase(containerType);

		Iterator<Object> containersIt = containers.descendingIterator();

		// We have always entered the current 'this' context when processing properties, so the first thing
		// we get will always be 'this'. So skip that and the first matching owner should be what we want.
		containersIt.next();

		while (containersIt.hasNext()) {
			Object potentialContainer = containersIt.next();

			if (containerClass.isAssignableFrom(potentialContainer.getClass()))
				return potentialContainer;
		}

		throw new IllegalStateException("No container matching " + containerType + " in " + containers + " at path " + path);
	}

Get the container object which is appropriate for the specified property. Go up the chain looking for a compatible
type; the first one found is the container. If nothing found, throw an exception.

Get the container object which is appropriate for the specified property . Go up the chain looking for a compatible type ; the first one found is the container . If nothing found throw an exception .


private Stat getStat(String kind)
	{
		Stat stat = this.stats.get(kind);
		if (stat == null)
		{
			stat = new Stat();
			this.stats.put(kind, stat);
		}

		return stat;
	}

private Stat getStat ( String kind ) { Stat stat = this . stats . get ( kind ) ; if ( stat == null ) { stat = new Stat ( ) ; this . stats . put ( kind , stat ) ; } return stat ; }

We're just tracking statistics so we don't really need to worry about these stepping on each other;
if there's a hit or miss lost no big deal.

We re just tracking statistics so we don t really need to worry about these stepping on each other ; if there s a hit or miss lost no big deal .


public static void unwrapAndThrow(Throwable ex)
	{
		if (ex instanceof RuntimeException)
			throw (RuntimeException)ex;
		else if (ex instanceof Error)
			throw (Error)ex;
		else if (ex instanceof ExecutionException)
			unwrapAndThrow(ex.getCause());
		else
			throw new UndeclaredThrowableException(ex);
	}

public static void unwrapAndThrow ( Throwable ex ) { if ( ex instanceof RuntimeException ) throw ( RuntimeException ) ex ; else if ( ex instanceof Error ) throw ( Error ) ex ; else if ( ex instanceof ExecutionException ) unwrapAndThrow ( ex . getCause ( ) ) ; else throw new UndeclaredThrowableException ( ex ) ; }

Properly unwraps ExecutionException, throwing the relevant original cause.  Otherwise
RuntimeExceptions get thrown and checked exceptions get wrapped in a RuntimeException.

Properly unwraps ExecutionException throwing the relevant original cause . Otherwise RuntimeExceptions get thrown and checked exceptions get wrapped in a RuntimeException .


public <P, D> Translator<P, D> get(final TypeKey tk, final CreateContext ctx, final Path path) {

		Translator<?, ?> translator = translators.get(tk);
		if (translator == null) {
			translator = create(tk, ctx, path);
			translators.put(tk, translator);
		}

		//noinspection unchecked
		return (Translator<P, D>)translator;
	}

public
<
P
,
D
>
Translator
<
P
,
D
>
get
(
final
TypeKey
tk
,
final
CreateContext
ctx
,
final
Path
path
)
{
Translator
<
,
>
translator
=
translators
.
get
(
tk
)
;
if
(
translator
==
)
{
translator
=
create
(
tk
,
ctx
,
path
)
;
translators
.
put
(
tk
,
translator
)
;
}
//noinspection unchecked
return
(
Translator
<
P
,
D
>
)
translator
;
}

Obtains the Translator appropriate for this type and annotations. May be a cached
translator; if not, one will be discovered and cached.

Obtains the Translator appropriate for this type and annotations . May be a cached translator ; if not one will be discovered and cached .


public <P> Translator<P, FullEntity<?>> getRoot(final Class<P> clazz) {
		return get(new TypeKey(clazz), new CreateContext(fact), Path.root());
	}

public < P > Translator < P , FullEntity < ? > > getRoot ( final Class < P > clazz ) { return get ( new TypeKey ( clazz ) , new CreateContext ( fact ) , Path . root ( ) ) ; }

Get the translator for a root entity class


private Translator<?, ?> create(final TypeKey tk, final CreateContext ctx, final Path path) {
		for (final TranslatorFactory<?, ?> trans: this.translatorFactories) {
			@SuppressWarnings("unchecked")
			final Translator<?, ?> soFar = trans.create(tk, ctx, path);
			if (soFar != null)
				return soFar;
		}

		throw new IllegalArgumentException("Don't know how to translate " + tk.getType() + " with annotations " + Arrays.toString(tk.getAnnotations()));
	}

private
Translator
<
,
>
create
(
final
TypeKey
tk
,
final
CreateContext
ctx
,
final
Path
path
)
{
for
(
final
TranslatorFactory
<
,
>
trans
:
this
.
translatorFactories
)
{
@
SuppressWarnings
(
"unchecked"
)
final
Translator
<
,
>
soFar
=
trans
.
create
(
tk
,
ctx
,
path
)
;
if
(
soFar
!=
)
return
soFar
;
}
throw
new
IllegalArgumentException
(
"Don't know how to translate "
+
tk
.
getType
(
)
+
" with annotations "
+
Arrays
.
toString
(
tk
.
getAnnotations
(
)
)
)
;
}

Create a translator from scratch by going through the discovery process.

Create a translator from scratch by going through the discovery process .


@Override
	public void load(final FullEntity<?> container, final LoadContext ctx, final Path containerPath, final P intoPojo) {
		try {
			if (translator instanceof Recycles)
				ctx.recycle(property.get(intoPojo));

			final Value<D> value = (translator instanceof Synthetic)
				? null
				: getPropertyFromContainer(container, containerPath);	// will throw SkipException if property not present

			setValue(intoPojo, value, ctx, containerPath);
		}
		catch (SkipException ex) {
			// Irrelevant
		}
	}

Gets the appropriate value from the container and sets it on the appropriate field of the pojo.

Gets the appropriate value from the container and sets it on the appropriate field of the pojo .


private Value<D> getPropertyFromContainer(final FullEntity<?> container, final Path containerPath) {
		String foundName = null;
		Value<D> value = null;

		for (String name: property.getLoadNames()) {
			if (container.contains(name)) {
				if (foundName != null)
					throw new IllegalStateException("Collision trying to load field; multiple name matches for '"
							+ property.getName() + "' at '" + containerPath.extend(foundName) + "' and '" + containerPath.extend(name) + "'");

				//noinspection unchecked
				value = container.getValue(name);
				foundName = name;
			}
		}

		if (foundName == null)
			throw new SkipException();
		else
			return value;
	}

Gets the relevant property from the container, detecting alsoload collisions.

@return the value obtained from the container
@throws IllegalStateException if there are multiple alsoload name matches

Gets the relevant property from the container detecting alsoload collisions .


public void setValue(final Object pojo, final Value<D> value, final LoadContext ctx, final Path containerPath) throws SkipException {
		final Path propertyPath = containerPath.extend(property.getName());
		final P loaded = translator.load(value, ctx, propertyPath);

		setOnPojo(pojo, loaded, ctx, propertyPath);
	}

public void setValue ( final Object pojo , final Value < D > value , final LoadContext ctx , final Path containerPath ) throws SkipException { final Path propertyPath = containerPath . extend ( property . getName ( ) ) ; final P loaded = translator . load ( value , ctx , propertyPath ) ; setOnPojo ( pojo , loaded , ctx , propertyPath ) ; }

Set this raw datastore value on the relevant property of the pojo, doing whatever translations are necessary.

Set this raw datastore value on the relevant property of the pojo doing whatever translations are necessary .


@Override
	public void save(final P onPojo, boolean index, final SaveContext ctx, final Path containerPath, final FullEntity.Builder<?> into) {
		if (property.isSaved(onPojo)) {
			// Look for an override on indexing
			final Boolean propertyIndexInstruction = property.getIndexInstruction(onPojo);
			if (propertyIndexInstruction != null)
				index = propertyIndexInstruction;

			@SuppressWarnings("unchecked")
			final P value = (P)property.get(onPojo);
			try {
				final Path propPath = containerPath.extend(property.getName());
				final Value<D> propValue = translator.save(value, index, ctx, propPath);

				into.set(property.getName(), propValue);
			}
			catch (SkipException ex) {
				// No problem, do nothing
			}
		}
	}

@
Override
public
void
save
(
final
P
onPojo
,
boolean
index
,
final
SaveContext
ctx
,
final
Path
containerPath
,
final
FullEntity
.
Builder
<
>
into
)
{
if
(
property
.
isSaved
(
onPojo
)
)
{
// Look for an override on indexing
final
Boolean
propertyIndexInstruction
=
property
.
getIndexInstruction
(
onPojo
)
;
if
(
propertyIndexInstruction
!=
)
index
=
propertyIndexInstruction
;
@
SuppressWarnings
(
"unchecked"
)
final
P
value
=
(
P
)
property
.
get
(
onPojo
)
;
try
{
final
Path
propPath
=
containerPath
.
extend
(
property
.
getName
(
)
)
;
final
Value
<
D
>
propValue
=
translator
.
save
(
value
,
index
,
ctx
,
propPath
)
;
into
.
set
(
property
.
getName
(
)
,
propValue
)
;
}
catch
(
SkipException
ex
)
{
// No problem, do nothing
}
}
}

Gets the appropriate field value from the pojo and puts it in the container at the appropriate prop name
and with the appropriate indexing.
@param onPojo is the parent pojo which holds the property we represent
@param index is the default state of indexing up to this point
@param containerPath is the path to the container; each property will extend this path.

Gets the appropriate field value from the pojo and puts it in the container at the appropriate prop name and with the appropriate indexing .


public Value<D> getValue(final Object pojo, final SaveContext ctx, final Path containerPath) {
		@SuppressWarnings("unchecked")
		final P value = (P)property.get(pojo);

		return translator.save(value, false, ctx, containerPath.extend(property.getName()));
	}

public Value < D > getValue ( final Object pojo , final SaveContext ctx , final Path containerPath ) { @ SuppressWarnings ( "unchecked" ) final P value = ( P ) property . get ( pojo ) ; return translator . save ( value , false , ctx , containerPath . extend ( property . getName ( ) ) ) ; }

Get the value for the property and translate it into datastore format.

Get the value for the property and translate it into datastore format .


@SuppressWarnings("unchecked")
  protected final <EM extends ExtendableMessage<EM>>
  void writeExtensions(EM msg, JsonGenerator gen) throws IOException {
    boolean openExt = false;

    for (Map.Entry<FieldDescriptor, Object> field : msg.getAllFields().entrySet()) {
      FieldDescriptor fd = field.getKey();
      if (fd.isExtension()) {
        if (fd.isRepeated()) {
          List<Object> extValue = (List<Object>) field.getValue();
          if (!extValue.isEmpty()) {
            OpenRtbJsonExtWriter<Object> extWriter =
                factory.getWriter(msg.getClass(), extValue.get(0).getClass(), fd.getName());
            if (extWriter != null) {
              openExt = openExt(gen, openExt);
              extWriter.writeRepeated(extValue, gen);
            }
          }
        } else {
          Object extValue = field.getValue();
          OpenRtbJsonExtWriter<Object> extWriter =
              factory.getWriter(msg.getClass(), extValue.getClass(), fd.getName());
          if (extWriter != null) {
            openExt = openExt(gen, openExt);
            extWriter.writeSingle(extValue, gen);
          }
        }
      }
    }

    if (openExt) {
      gen.writeEndObject();
    }
  }

@ SuppressWarnings ( "unchecked" ) protected final < EM extends ExtendableMessage < EM > > void writeExtensions ( EM msg , JsonGenerator gen ) throws IOException { boolean openExt = false ; for ( Map . Entry < FieldDescriptor , Object > field : msg . getAllFields ( ) . entrySet ( ) ) { FieldDescriptor fd = field . getKey ( ) ; if ( fd . isExtension ( ) ) { if ( fd . isRepeated ( ) ) { List < Object > extValue = ( List < Object > ) field . getValue ( ) ; if ( ! extValue . isEmpty ( ) ) { OpenRtbJsonExtWriter < Object > extWriter = factory . getWriter ( msg . getClass ( ) , extValue . get ( 0 ) . getClass ( ) , fd . getName ( ) ) ; if ( extWriter != null ) { openExt = openExt ( gen , openExt ) ; extWriter . writeRepeated ( extValue , gen ) ; } } } else { Object extValue = field . getValue ( ) ; OpenRtbJsonExtWriter < Object > extWriter = factory . getWriter ( msg . getClass ( ) , extValue . getClass ( ) , fd . getName ( ) ) ; if ( extWriter != null ) { openExt = openExt ( gen , openExt ) ; extWriter . writeSingle ( extValue , gen ) ; } } } } if ( openExt ) { gen . writeEndObject ( ) ; } }

Write any extensions that may exist in a message.

@param msg A message that may contain extensions
@param gen The JSON generator
@param <EM> Type of message being serialized, containing extensions
@throws IOException any serialization error

Write any extensions that may exist in a message .


protected final boolean writeContentCategory(String cat, JsonGenerator gen) throws IOException {
    if (!factory.isStrict() || OpenRtbUtils.categoryFromName(cat) != null) {
      gen.writeString(cat);
      return true;
    } else {
      return false;
    }
  }

protected final boolean writeContentCategory ( String cat , JsonGenerator gen ) throws IOException { if ( ! factory . isStrict ( ) || OpenRtbUtils . categoryFromName ( cat ) != null ) { gen . writeString ( cat ) ; return true ; } else { return false ; } }

Writes a string that represents a ContentCategory's JSON name, returning success status.
If the factory is in strict mode, the category name is validated.

Writes a string that represents a ContentCategory s JSON name returning success status . If the factory is in strict mode the category name is validated .


protected final void writeContentCategories(
      String fieldName, List<String> cats, JsonGenerator gen)
      throws IOException {
    if (!cats.isEmpty()) {
      gen.writeArrayFieldStart(fieldName);
      for (String cat : cats) {
        writeContentCategory(cat, gen);
      }
      gen.writeEndArray();
    }
  }

protected final void writeContentCategories ( String fieldName , List < String > cats , JsonGenerator gen ) throws IOException { if ( ! cats . isEmpty ( ) ) { gen . writeArrayFieldStart ( fieldName ) ; for ( String cat : cats ) { writeContentCategory ( cat , gen ) ; } gen . writeEndArray ( ) ; } }

Writes an array of ContentCategory if not empty.

@see #writeContentCategory(String, JsonGenerator)

Writes an array of ContentCategory if not empty .


public String process(SnippetProcessorContext ctx, String snippet) {
    checkNotNull(ctx);
    StringBuilder sb = ctx.builder();
    sb.setLength(0);
    String currSnippet = snippet;

    boolean processedMacros = false;
    int snippetPos = 0;
    int macroPos = currSnippet.indexOf("${");

    while (macroPos != -1) {
      sb.append(currSnippet.substring(snippetPos, macroPos));
      int macroEnd = processMacroAt(ctx, currSnippet, macroPos);

      if (macroEnd == -1) {
        sb.append("${");
        snippetPos = macroPos + 2;
      } else {
        snippetPos = macroEnd;
        processedMacros = true;
      }

      macroPos = currSnippet.indexOf("${", snippetPos);
    }

    if (processedMacros) {
      sb.append(currSnippet, snippetPos, currSnippet.length());
      currSnippet = sb.toString();
    }
    sb.setLength(0);

    String ret = urlEncode(ctx, currSnippet);
    sb.setLength(0);
    return ret;
  }

public String process ( SnippetProcessorContext ctx , String snippet ) { checkNotNull ( ctx ) ; StringBuilder sb = ctx . builder ( ) ; sb . setLength ( 0 ) ; String currSnippet = snippet ; boolean processedMacros = false ; int snippetPos = 0 ; int macroPos = currSnippet . indexOf ( "${" ) ; while ( macroPos != - 1 ) { sb . append ( currSnippet . substring ( snippetPos , macroPos ) ) ; int macroEnd = processMacroAt ( ctx , currSnippet , macroPos ) ; if ( macroEnd == - 1 ) { sb . append ( "${" ) ; snippetPos = macroPos + 2 ; } else { snippetPos = macroEnd ; processedMacros = true ; } macroPos = currSnippet . indexOf ( "${" , snippetPos ) ; } if ( processedMacros ) { sb . append ( currSnippet , snippetPos , currSnippet . length ( ) ) ; currSnippet = sb . toString ( ) ; } sb . setLength ( 0 ) ; String ret = urlEncode ( ctx , currSnippet ) ; sb . setLength ( 0 ) ; return ret ; }

Processes the raw snippet that was set by the bid, making any transformations necessary.

Processes the raw snippet that was set by the bid making any transformations necessary .


public static Iterable<Bid.Builder> bids(BidResponse.Builder response) {
    return new ResponseBidsIterator(response, SEAT_ANY, null);
  }

public static Iterable < Bid . Builder > bids ( BidResponse . Builder response ) { return new ResponseBidsIterator ( response , SEAT_ANY , null ) ; }

Iterates all bids.

@return Read-only sequence of all bis in the response.
May have bids from multiple seats, grouped by seat


@Nullable public static Bid.Builder bidWithId(BidResponse.Builder response, String id) {
    checkNotNull(id);

    for (SeatBid.Builder seatbid : response.getSeatbidBuilderList()) {
      for (Bid.Builder bid : seatbid.getBidBuilderList()) {
        if (id.equals(bid.getId())) {
          return bid;
        }
      }
    }
    return null;
  }

@ Nullable public static Bid . Builder bidWithId ( BidResponse . Builder response , String id ) { checkNotNull ( id ) ; for ( SeatBid . Builder seatbid : response . getSeatbidBuilderList ( ) ) { for ( Bid . Builder bid : seatbid . getBidBuilderList ( ) ) { if ( id . equals ( bid . getId ( ) ) ) { return bid ; } } } return null ; }

Finds a bid by ID.

@param id Bid ID, assumed to be unique within the response
@return Matching bid's builder, or {@code null} if not found


public static boolean updateBids(
      BidResponse.Builder response, Function<Bid.Builder, Boolean> updater) {
    checkNotNull(updater);

    boolean updated = false;
    for (SeatBid.Builder seatbid : response.getSeatbidBuilderList()) {
      updated |= ProtoUtils.update(seatbid.getBidBuilderList(), updater);
    }
    return updated;
  }

public static boolean updateBids ( BidResponse . Builder response , Function < Bid . Builder , Boolean > updater ) { checkNotNull ( updater ) ; boolean updated = false ; for ( SeatBid . Builder seatbid : response . getSeatbidBuilderList ( ) ) { updated |= ProtoUtils . update ( seatbid . getBidBuilderList ( ) , updater ) ; } return updated ; }

Updates bids, from all seats.

@param updater Update function. The {@code apply()} method can decide or not to update each
object, and it's expected to return {@code true} for objects that were updated
@return {@code true} if at least one bid was updated
@see ProtoUtils#update(Iterable, Function) for more general updating support


public static boolean removeBids(BidResponse.Builder response, Predicate<Bid.Builder> filter) {
    checkNotNull(filter);
    boolean updated = false;

    for (SeatBid.Builder seatbid : response.getSeatbidBuilderList()) {
      updated |= removeBids(seatbid, filter);
    }

    return updated;
  }

public static boolean removeBids ( BidResponse . Builder response , Predicate < Bid . Builder > filter ) { checkNotNull ( filter ) ; boolean updated = false ; for ( SeatBid . Builder seatbid : response . getSeatbidBuilderList ( ) ) { updated |= removeBids ( seatbid , filter ) ; } return updated ; }

Remove bids by bid.

@param filter Returns {@code true} to keep bid, {@code false} to remove
@return {@code true} if any bid was removed
@see ProtoUtils#filter(List, Predicate) for more general filtering support


public static Iterable<Imp> impsWith(BidRequest request, Predicate<Imp> impFilter) {
    checkNotNull(impFilter);

    List<Imp> imps = request.getImpList();
    if (imps.isEmpty() || impFilter == IMP_ALL) {
      return imps;
    } else if (impFilter == IMP_NONE) {
      return ImmutableList.of();
    }

    boolean included = impFilter.test(imps.get(0));
    int size = imps.size(), i;

    for (i = 1; i < size; ++i) {
      if (impFilter.test(imps.get(i)) != included) {
        break;
      }
    }

    if (i == size) {
      return included
          ? imps // Unmodifiable, comes from protobuf
          : ImmutableList.<Imp>of();
    }

    int headingSize = i;
    return new FluentIterable<Imp>() {
      @Override public Iterator<Imp> iterator() {
        Iterator<Imp> unfiltered = imps.iterator();
        return new AbstractIterator<Imp>() {
          private int heading = 0;
          @Override protected Imp computeNext() {
            while (unfiltered.hasNext()) {
              Imp imp = unfiltered.next();
              if ((heading++ < headingSize)
                  ? included
                  : impFilter.test(imp)) {
                return imp;
              }
            }
            return endOfData();
        }};
      }};
  }

public
static
Iterable
<
Imp
>
impsWith
(
BidRequest
request
,
Predicate
<
Imp
>
impFilter
)
{
checkNotNull
(
impFilter
)
;
List
<
Imp
>
imps
=
request
.
getImpList
(
)
;
if
(
imps
.
isEmpty
(
)
||
impFilter
==
IMP_ALL
)
{
return
imps
;
}
else
if
(
impFilter
==
IMP_NONE
)
{
return
ImmutableList
.
of
(
)
;
}
boolean
included
=
impFilter
.
test
(
imps
.
get
(
0
)
)
;
int
size
=
imps
.
size
(
)
,
i
;
for
(
i
=
1
;
i
<
size
;
++
i
)
{
if
(
impFilter
.
test
(
imps
.
get
(
i
)
)
!=
included
)
{
break
;
}
}
if
(
i
==
size
)
{
return
included
imps
// Unmodifiable, comes from protobuf
:
ImmutableList
.
<
Imp
>
of
(
)
;
}
int
headingSize
=
i
;
return
new
FluentIterable
<
Imp
>
(
)
{
@
Override
public
Iterator
<
Imp
>
iterator
(
)
{
Iterator
<
Imp
>
unfiltered
=
imps
.
iterator
(
)
;
return
new
AbstractIterator
<
Imp
>
(
)
{
private
int
heading
=
0
;
@
Override
protected
Imp
computeNext
(
)
{
while
(
unfiltered
.
hasNext
(
)
)
{
Imp
imp
=
unfiltered
.
next
(
)
;
if
(
(
heading
++
<
headingSize
)
included
:
impFilter
.
test
(
imp
)
)
{
return
imp
;
}
}
return
endOfData
(
)
;
}
}
;
}
}
;
}

Optimized code for most filtered lookups. This is worth the effort
because bidder code may invoke these lookup methods intensely;
common cases like everything-filtered or nothing-filtered are very dominant;
and simpler code previously used needed lots of temporary collections.

@param request Container of impressions
@param impFilter Filters impressions; will be executed exactly once,
and only for impressions that pass the banner/video type filters.
The constants {@link #IMP_NONE} and  {@link #IMP_ALL} allow
more efficient execution when you want to filter none/all impressions.
@return Immutable or unmodifiable view for the filtered impressions

Optimized code for most filtered lookups . This is worth the effort because bidder code may invoke these lookup methods intensely ; common cases like everything - filtered or nothing - filtered are very dominant ; and simpler code previously used needed lots of temporary collections .


protected final <EB extends ExtendableBuilder<?, EB>>
  void readExtensions(EB msg, JsonParser par) throws IOException {
    @SuppressWarnings("unchecked")
    Set<OpenRtbJsonExtReader<EB>> extReaders = factory.getReaders((Class<EB>) msg.getClass());
    if (extReaders.isEmpty()) {
      par.skipChildren();
      return;
    }

    startObject(par);
    JsonToken tokLast = par.getCurrentToken();
    JsonLocation locLast = par.getCurrentLocation();

    while (true) {
      boolean extRead = false;
      for (OpenRtbJsonExtReader<EB> extReader : extReaders) {
        if (extReader.filter(par)) {
          extReader.read(msg, par);
          JsonToken tokNew = par.getCurrentToken();
          JsonLocation locNew = par.getCurrentLocation();
          boolean advanced = tokNew != tokLast || !locNew.equals(locLast);
          extRead |= advanced;

          if (!endObject(par)) {
            return;
          } else if (advanced && par.getCurrentToken() != JsonToken.FIELD_NAME) {
            tokLast = par.nextToken();
            locLast = par.getCurrentLocation();
          } else {
            tokLast = tokNew;
            locLast = locNew;
          }
        }
      }

      if (!endObject(par)) {
        // Can't rely on this exit condition inside the for loop because no readers may filter.
        return;
      }

      if (!extRead) {
        // No field was consumed by any reader, so we need to skip the field to make progress.
        if (logger.isDebugEnabled()) {
          logger.debug("Extension field not consumed by any reader, skipping: {} @{}:{}",
              par.getCurrentName(), locLast.getLineNr(), locLast.getCharOffset());
        }
        par.nextToken();
        par.skipChildren();
        tokLast = par.nextToken();
        locLast = par.getCurrentLocation();
      }
      // Else loop, try all readers again
    }
  }

Read any extensions that may exist in a message.

@param msg Builder of a message that may contain extensions
@param par The JSON parser, positioned at the "ext" field
@param <EB> Type of message builder being constructed
@throws IOException any parsing error

Read any extensions that may exist in a message .


protected final boolean emptyToNull(JsonParser par) throws IOException {
    JsonToken token = par.getCurrentToken();
    if (token == null) {
      token = par.nextToken();
    }
    return !factory().isStrict() && token == null;
  }

protected final boolean emptyToNull ( JsonParser par ) throws IOException { JsonToken token = par . getCurrentToken ( ) ; if ( token == null ) { token = par . nextToken ( ) ; } return ! factory ( ) . isStrict ( ) && token == null ; }

Special case for empty-string input. Returning null in non-@Nullable method,
but this is non-strict mode anyway.

Special case for empty - string input . Returning null in non -


public final <EB extends ExtendableBuilder<?, EB>> OpenRtbJsonFactory register(
      OpenRtbJsonExtReader<EB> extReader, Class<EB> msgKlass) {
    extReaders.put(msgKlass.getName(), extReader);
    return this;
  }

public final < EB extends ExtendableBuilder < ? , EB > > OpenRtbJsonFactory register ( OpenRtbJsonExtReader < EB > extReader , Class < EB > msgKlass ) { extReaders . put ( msgKlass . getName ( ) , extReader ) ; return this ; }

Register an extension reader.

@param extReader code to desserialize some extension properties
@param msgKlass class of extension message's builder, e.g. {@code MyImp.Builder.class}


public final <T> OpenRtbJsonFactory register(OpenRtbJsonExtWriter<T> extWriter,
      Class<T> extKlass, Class<? extends Message> msgKlass, String fieldName) {
    Map<String, Map<String, OpenRtbJsonExtWriter<?>>> mapMsg = extWriters.get(msgKlass.getName());
    if (mapMsg == null) {
      extWriters.put(msgKlass.getName(), mapMsg = new LinkedHashMap<>());
    }
    Map<String, OpenRtbJsonExtWriter<?>> mapKlass = mapMsg.get(extKlass.getName());
    if (mapKlass == null) {
      mapMsg.put(extKlass.getName(), mapKlass = new LinkedHashMap<>());
    }
    mapKlass.put(fieldName == null ? FIELDNAME_ALL : fieldName, extWriter);
    return this;
  }

public final < T > OpenRtbJsonFactory register ( OpenRtbJsonExtWriter < T > extWriter , Class < T > extKlass , Class < ? extends Message > msgKlass , String fieldName ) { Map < String , Map < String , OpenRtbJsonExtWriter < ? > > > mapMsg = extWriters . get ( msgKlass . getName ( ) ) ; if ( mapMsg == null ) { extWriters . put ( msgKlass . getName ( ) , mapMsg = new LinkedHashMap <> ( ) ) ; } Map < String , OpenRtbJsonExtWriter < ? > > mapKlass = mapMsg . get ( extKlass . getName ( ) ) ; if ( mapKlass == null ) { mapMsg . put ( extKlass . getName ( ) , mapKlass = new LinkedHashMap <> ( ) ) ; } mapKlass . put ( fieldName == null ? FIELDNAME_ALL : fieldName , extWriter ) ; return this ; }

Register an extension writer, bound to a specific field name. This writer will be
used in preference to a non-field-specific writer that may exist for the same class.

@param extWriter code to serialize some {@code extKlass}'s properties
@param extKlass class of extension message, e.g. {@code MyImp.class}
@param msgKlass class of container message, e.g. {@code Imp.class}
@param fieldName name of the field containing the extension
@param <T> Type of value for the extension
@see #register(OpenRtbJsonExtWriter, Class, Class)

Register an extension writer bound to a specific field name . This writer will be used in preference to a non - field - specific writer that may exist for the same class .


public void process(SnippetProcessorContext bidCtx) {
    for (SeatBid.Builder seat : bidCtx.response().getSeatbidBuilderList()) {
      for (Bid.Builder bid : seat.getBidBuilderList()) {
        bidCtx.setBid(bid);
        processFields(bidCtx);
      }
    }
  }

public void process ( SnippetProcessorContext bidCtx ) { for ( SeatBid . Builder seat : bidCtx . response ( ) . getSeatbidBuilderList ( ) ) { for ( Bid . Builder bid : seat . getBidBuilderList ( ) ) { bidCtx . setBid ( bid ) ; processFields ( bidCtx ) ; } } }

Processes the context's response in-place, modifying properties that may contain macros.

Processes the context s response in - place modifying properties that may contain macros .


protected void processFields(SnippetProcessorContext bidCtx) {
    Bid.Builder bid = bidCtx.getBid();

    // Properties that can also be in the RHS of macros used by other properties.

    if (extendedFields) {
      if (bid.hasAdid()) {
        bid.setAdid(process(bidCtx, bid.getAdid()));
      }
      bid.setId(process(bidCtx, bid.getId()));
    }

    // Properties that are NOT the RHS of any macro.

    if (bid.hasAdm()) {
      bid.setAdm(process(bidCtx, bid.getAdm()));
    }

    if (extendedFields) {
      if (bid.hasBurl()) {
        bid.setBurl(process(bidCtx, bid.getBurl()));
      }
      if (bid.hasCid()) {
        bid.setCid(process(bidCtx, bid.getCid()));
      }
      if (bid.hasCrid()) {
        bid.setCrid(process(bidCtx, bid.getCrid()));
      }
      if (bid.hasDealid()) {
        bid.setDealid(process(bidCtx, bid.getDealid()));
      }
      bid.setImpid(process(bidCtx, bid.getImpid()));
      if (bid.hasIurl()) {
        bid.setIurl(process(bidCtx, bid.getIurl()));
      }
      if (bid.hasLurl()) {
        bid.setIurl(process(bidCtx, bid.getLurl()));
      }
      if (bid.hasNurl()) {
        bid.setNurl(process(bidCtx, bid.getNurl()));
      }
    }
  }

Processes all fields of a bid that should support macro expansion.

Processes all fields of a bid that should support macro expansion .


public static <B extends MessageLite.Builder> boolean update(
      Iterable<B> objs, Function<B, Boolean> updater) {
    checkNotNull(updater);
    boolean updated = false;

    for (B obj : objs) {
      updated |= updater.apply(obj);
    }
    return updated;
  }

public static < B extends MessageLite . Builder > boolean update ( Iterable < B > objs , Function < B , Boolean > updater ) { checkNotNull ( updater ) ; boolean updated = false ; for ( B obj : objs ) { updated |= updater . apply ( obj ) ; } return updated ; }

Updates every builder from a sequence.

@param objs List of builders to update
@param updater Update function. The {@code apply()} method can decide or not to update each
object, and it's expected to return {@code true} if some update was made
@return {@code true} if at least one object was updated


public static <M extends MessageLiteOrBuilder>
      List<M> filter(List<M> objs, Predicate<M> filter) {
    checkNotNull(filter);

    for (int i = 0; i < objs.size(); ++i) {
      if (!filter.test(objs.get(i))) {
        // At least one discarded object, go to slow-path.
        return filterFrom(objs, filter, i);
      }
    }

    // Optimized common case: all items filtered, return the input sequence.
    return objs;
  }

Runs a filter through a sequence of objects.

@param objs Message-or-builder objects
@param filter Function that returns {@code true} to retain an object, {@code false} to discard
@return Retained objects. If some elements are retained and others are discarded,
this will be a new, mutable {@link List} that contains only the retained elements.
If all elements are retained, returns the same, unmodified input sequence.
If all elements are discarded, returns an immutable, empty sequence

Runs a filter through a sequence of objects .


public static String getCurrentName(JsonParser par) throws IOException {
    String name = par.getCurrentName();
    return name == null ? "" : name;
  }

public static String getCurrentName ( JsonParser par ) throws IOException { String name = par . getCurrentName ( ) ; return name == null ? "" : name ; }

Returns the current field name, or empty string if none.

Returns the current field name or empty string if none .


public static void startObject(JsonParser par) throws IOException {
    JsonToken token = par.getCurrentToken();
    if (token == null || token == JsonToken.FIELD_NAME) {
      token = par.nextToken();
    }
    if (token == JsonToken.START_OBJECT) {
      par.nextToken();
    } else {
      throw new JsonParseException(par, "Expected start of object");
    }
  }

public
static
void
startObject
(
JsonParser
par
)
throws
IOException
{
JsonToken
token
=
par
.
getCurrentToken
(
)
;
if
(
token
==
||
token
==
JsonToken
.
FIELD_NAME
)
{
token
=
par
.
nextToken
(
)
;
}
if
(
token
==
JsonToken
.
START_OBJECT
)
{
par
.
nextToken
(
)
;
}
else
{
throw
new
JsonParseException
(
par
,
"Expected start of object"
)
;
}
}

Starts an Object, skipping the '{' token, and if necessary a field name before it.

Starts an Object skipping the { token and if necessary a field name before it .


public static void startArray(JsonParser par) throws IOException {
    JsonToken token = par.getCurrentToken();
    if (token == null || token == JsonToken.FIELD_NAME) {
      token = par.nextToken();
    }
    if (token == JsonToken.START_ARRAY) {
      par.nextToken();
    } else {
      throw new JsonParseException(par, "Expected start of array");
    }
  }

public
static
void
startArray
(
JsonParser
par
)
throws
IOException
{
JsonToken
token
=
par
.
getCurrentToken
(
)
;
if
(
token
==
||
token
==
JsonToken
.
FIELD_NAME
)
{
token
=
par
.
nextToken
(
)
;
}
if
(
token
==
JsonToken
.
START_ARRAY
)
{
par
.
nextToken
(
)
;
}
else
{
throw
new
JsonParseException
(
par
,
"Expected start of array"
)
;
}
}

Starts an Array, skipping the '[' token, and if necessary a field name before it.

Starts an Array skipping the [ token and if necessary a field name before it .


public static JsonToken peekToken(JsonParser par) throws IOException {
    JsonToken token = par.getCurrentToken();
    if (token == null || token == JsonToken.FIELD_NAME) {
      token = par.nextToken();
    }
    return token;
  }

public static JsonToken peekToken ( JsonParser par ) throws IOException { JsonToken token = par . getCurrentToken ( ) ; if ( token == null || token == JsonToken . FIELD_NAME ) { token = par . nextToken ( ) ; } return token ; }

Skips a field name if necessary, returning the current token then.

Skips a field name if necessary returning the current token then .


public static void writeIntBoolField(String fieldName, boolean data, JsonGenerator gen)
      throws IOException {
    gen.writeNumberField(fieldName, data ? 1 : 0);
  }

public static void writeIntBoolField ( String fieldName , boolean data , JsonGenerator gen ) throws IOException { gen . writeNumberField ( fieldName , data ? 1 : 0 ) ; }

Writes a boolean as int, where false = 0 and true = 1.

Writes a boolean as int where false = 0 and true = 1 .


public static void writeStrings(String fieldName, List<String> data, JsonGenerator gen)
      throws IOException {
    if (!data.isEmpty()) {
      gen.writeArrayFieldStart(fieldName);
      for (String d : data) {
        gen.writeString(d);
      }
      gen.writeEndArray();
    }
  }

public static void writeStrings ( String fieldName , List < String > data , JsonGenerator gen ) throws IOException { if ( ! data . isEmpty ( ) ) { gen . writeArrayFieldStart ( fieldName ) ; for ( String d : data ) { gen . writeString ( d ) ; } gen . writeEndArray ( ) ; } }


public static void writeInts(String fieldName, List<Integer> data, JsonGenerator gen)
      throws IOException {
    if (!data.isEmpty()) {
      gen.writeArrayFieldStart(fieldName);
      for (Integer d : data) {
        gen.writeNumber(d);
      }
      gen.writeEndArray();
    }
  }

public static void writeInts ( String fieldName , List < Integer > data , JsonGenerator gen ) throws IOException { if ( ! data . isEmpty ( ) ) { gen . writeArrayFieldStart ( fieldName ) ; for ( Integer d : data ) { gen . writeNumber ( d ) ; } gen . writeEndArray ( ) ; } }


public static void writeLongs(String fieldName, List<Long> data, JsonGenerator gen)
      throws IOException {
    if (!data.isEmpty()) {
      gen.writeArrayFieldStart(fieldName);
      for (long d : data) {
        writeLong(d, gen);
      }
      gen.writeEndArray();
    }
  }

public static void writeLongs ( String fieldName , List < Long > data , JsonGenerator gen ) throws IOException { if ( ! data . isEmpty ( ) ) { gen . writeArrayFieldStart ( fieldName ) ; for ( long d : data ) { writeLong ( d , gen ) ; } gen . writeEndArray ( ) ; } }

Writes a long array if not empty, using quotes for values that are too big.

@see #writeLong(long, JsonGenerator)

Writes a long array if not empty using quotes for values that are too big .


public static void writeEnums(
      String fieldName, List<? extends ProtocolMessageEnum> enums, JsonGenerator gen)
      throws IOException {
    if (!enums.isEmpty()) {
      gen.writeArrayFieldStart(fieldName);
      for (ProtocolMessageEnum e : enums) {
        writeEnum(e, gen);
      }
      gen.writeEndArray();
    }
  }

public static void writeEnums ( String fieldName , List < ? extends ProtocolMessageEnum > enums , JsonGenerator gen ) throws IOException { if ( ! enums . isEmpty ( ) ) { gen . writeArrayFieldStart ( fieldName ) ; for ( ProtocolMessageEnum e : enums ) { writeEnum ( e , gen ) ; } gen . writeEndArray ( ) ; } }

Writes a enum array if not empty.

@see #writeEnum(ProtocolMessageEnum, JsonGenerator)


public static URL resolveURL(URL base, String target)
            throws MalformedURLException {
        target = target.trim();

        if (target.startsWith("?")) {
            return fixPureQueryTargets(base, target);
        }

        return new URL(base, target);
    }

public static URL resolveURL ( URL base , String target ) throws MalformedURLException { target = target . trim ( ) ; if ( target . startsWith ( "?" ) ) { return fixPureQueryTargets ( base , target ) ; } return new URL ( base , target ) ; }

Resolve relative URL-s and fix a few java.net.URL errors in handling of
URLs with embedded params and pure query targets.

@param base
base url
@param target
target url (may be relative)
@return resolved absolute url.
@throws MalformedURLException

Resolve relative URL - s and fix a few java . net . URL errors in handling of URLs with embedded params and pure query targets .


static URL fixPureQueryTargets(URL base, String target)
            throws MalformedURLException {
        if (!target.startsWith("?"))
            return new URL(base, target);

        String basePath = base.getPath();
        String baseRightMost = "";
        int baseRightMostIdx = basePath.lastIndexOf("/");
        if (baseRightMostIdx != -1) {
            baseRightMost = basePath.substring(baseRightMostIdx + 1);
        }

        if (target.startsWith("?"))
            target = baseRightMost + target;

        return new URL(base, target);
    }

static URL fixPureQueryTargets ( URL base , String target ) throws MalformedURLException { if ( ! target . startsWith ( "?" ) ) return new URL ( base , target ) ; String basePath = base . getPath ( ) ; String baseRightMost = "" ; int baseRightMostIdx = basePath . lastIndexOf ( "/" ) ; if ( baseRightMostIdx != - 1 ) { baseRightMost = basePath . substring ( baseRightMostIdx + 1 ) ; } if ( target . startsWith ( "?" ) ) target = baseRightMost + target ; return new URL ( base , target ) ; }

Handle the case in RFC3986 section 5.4.1 example 7, and similar.

Handle the case in RFC3986 section 5 . 4 . 1 example 7 and similar .


public static String[] getHostSegments(URL url) {
        String host = url.getHost();
        // return whole hostname, if it is an ipv4
        // TODO : handle ipv6
        if (IP_PATTERN.matcher(host).matches())
            return new String[] { host };
        return host.split("\\.");
    }

Partitions of the hostname of the url by "."

Partitions of the hostname of the url by .


public static String getHost(String url) {
        try {
            return new URL(url).getHost().toLowerCase(Locale.ROOT);
        } catch (MalformedURLException e) {
            return null;
        }
    }

public static String getHost ( String url ) { try { return new URL ( url ) . getHost ( ) . toLowerCase ( Locale . ROOT ) ; } catch ( MalformedURLException e ) { return null ; } }

Returns the lowercased hostname for the url or null if the url is not
well formed.

@param url
The url to check.
@return String The hostname for the url.

Returns the lowercased hostname for the url or null if the url is not well formed .


public static String getPage(String url) {
        try {
            // get the full url, and replace the query string with and empty
            // string
            url = url.toLowerCase(Locale.ROOT);
            String queryStr = new URL(url).getQuery();
            return (queryStr != null) ? url.replace("?" + queryStr, "") : url;
        } catch (MalformedURLException e) {
            return null;
        }
    }

Returns the page for the url. The page consists of the protocol, host,
and path, but does not include the query string. The host is lowercased
but the path is not.

@param url
The url to check.
@return String The page for the url.

Returns the page for the url . The page consists of the protocol host and path but does not include the query string . The host is lowercased but the path is not .


public static List<String> loadListFromConf(String paramKey, Map stormConf) {
        Object obj = stormConf.get(paramKey);
        List<String> list = new LinkedList<>();

        if (obj == null)
            return list;

        if (obj instanceof PersistentVector) {
            list.addAll((PersistentVector) obj);
        } else { // single value?
            list.add(obj.toString());
        }
        return list;
    }

public
static
List
<
String
>
loadListFromConf
(
String
paramKey
,
Map
stormConf
)
{
Object
obj
=
stormConf
.
get
(
paramKey
)
;
List
<
String
>
list
=
new
LinkedList
<>
(
)
;
if
(
obj
==
)
return
list
;
if
(
obj
instanceof
PersistentVector
)
{
list
.
addAll
(
(
PersistentVector
)
obj
)
;
}
else
{
// single value?
list
.
add
(
obj
.
toString
(
)
)
;
}
return
list
;
}

Return one or more Strings regardless of whether they are represented as
a single String or a list in the config or an empty List if no value
could be found for that key.

Return one or more Strings regardless of whether they are represented as a single String or a list in the config or an empty List if no value could be found for that key .


public static Map extractConfigElement(Map conf) {
        if (conf.size() == 1) {
            Object confNode = conf.get("config");
            if (confNode != null && confNode instanceof Map) {
                conf = (Map) confNode;
            }
        }
        return conf;
    }

public static Map extractConfigElement ( Map conf ) { if ( conf . size ( ) == 1 ) { Object confNode = conf . get ( "config" ) ; if ( confNode != null && confNode instanceof Map ) { conf = ( Map ) confNode ; } } return conf ; }

If the config consists of a single key 'config', its values are used
instead

If the config consists of a single key config its values are used instead


public synchronized Protocol getProtocol(URL url) {
        // get the protocol
        String protocol = url.getProtocol();
        return cache.get(protocol);
    }

public
synchronized
Protocol
getProtocol
(
URL
url
)
{
// get the protocol
String
protocol
=
url
.
getProtocol
(
)
;
return
cache
.
get
(
protocol
)
;
}

Returns an instance of the protocol to use for a given URL


public static byte[] generateWARCInfo(Map<String, String> fields) {
        StringBuffer buffer = new StringBuffer();
        buffer.append(WARC_VERSION);
        buffer.append(CRLF);

        buffer.append("WARC-Type: warcinfo").append(CRLF);

        String mainID = UUID.randomUUID().toString();

        // retrieve the date and filename from the map
        String date = fields.get("WARC-Date");
        buffer.append("WARC-Date: ").append(date).append(CRLF);

        String filename = fields.get("WARC-Filename");
        buffer.append("WARC-Filename: ").append(filename).append(CRLF);

        buffer.append("WARC-Record-ID").append(": ").append("<urn:uuid:")
                .append(mainID).append(">").append(CRLF);

        buffer.append("Content-Type").append(": ")
                .append("application/warc-fields").append(CRLF);

        StringBuilder fieldsBuffer = new StringBuilder();

        // add WARC fields
        // http://bibnum.bnf.fr/warc/WARC_ISO_28500_version1_latestdraft.pdf
        Iterator<Entry<String, String>> iter = fields.entrySet().iterator();
        while (iter.hasNext()) {
            Entry<String, String> entry = iter.next();
            String key = entry.getKey();
            if (key.startsWith("WARC-"))
                continue;
            fieldsBuffer.append(key).append(": ").append(entry.getValue())
                    .append(CRLF);
        }

        buffer.append("Content-Length")
                .append(": ")
                .append(fieldsBuffer.toString()
                        .getBytes(StandardCharsets.UTF_8).length).append(CRLF);

        buffer.append(CRLF);

        buffer.append(fieldsBuffer.toString());

        buffer.append(CRLF);
        buffer.append(CRLF);

        return buffer.toString().getBytes(StandardCharsets.UTF_8);
    }

Generates a WARC info entry which can be stored at the beginning of each
WARC file.

Generates a WARC info entry which can be stored at the beginning of each WARC file .


public static String extractRefreshURL(String value) {
        if (StringUtils.isBlank(value))
            return null;

        // 0;URL=http://www.apollocolors.com/site
        try {
            if (matcher.reset(value).matches()) {
                return matcher.group(1);
            }
        } catch (Exception e) {
        }

        return null;
    }

Returns a normalised value of the content attribute for the refresh tag


public Metadata getMetaForOutlink(String targetURL, String sourceURL,
            Metadata parentMD) {
        Metadata md = _filter(parentMD, mdToTransfer);

        // keep the path?
        if (trackPath) {
            md.addValue(urlPathKeyName, sourceURL);
        }

        // track depth
        if (trackDepth) {
            String existingDepth = md.getFirstValue(depthKeyName);
            int depth;
            try {
                depth = Integer.parseInt(existingDepth);
            } catch (Exception e) {
                depth = 0;
            }
            md.setValue(depthKeyName, Integer.toString(++depth));
        }

        return md;
    }

public
Metadata
getMetaForOutlink
(
String
targetURL
,
String
sourceURL
,
Metadata
parentMD
)
{
Metadata
md
=
_filter
(
parentMD
,
mdToTransfer
)
;
// keep the path?
if
(
trackPath
)
{
md
.
addValue
(
urlPathKeyName
,
sourceURL
)
;
}
// track depth
if
(
trackDepth
)
{
String
existingDepth
=
md
.
getFirstValue
(
depthKeyName
)
;
int
depth
;
try
{
depth
=
Integer
.
parseInt
(
existingDepth
)
;
}
catch
(
Exception
e
)
{
depth
=
0
;
}
md
.
setValue
(
depthKeyName
,
Integer
.
toString
(
++
depth
)
)
;
}
return
md
;
}

Determine which metadata should be transfered to an outlink. Adds
additional metadata like the URL path.

Determine which metadata should be transfered to an outlink . Adds additional metadata like the URL path .


public Metadata filter(Metadata metadata) {
        Metadata filtered_md = _filter(metadata, mdToTransfer);

        // add the features that are only persisted but
        // not transfered like __redirTo_
        filtered_md.putAll(_filter(metadata, mdToPersistOnly));

        return filtered_md;
    }

Determine which metadata should be persisted for a given document
including those which are not necessarily transferred to the outlinks


public static void add(String url, Metadata md, Date nextFetch) {
        LOG.debug("Adding {} with md {} and nextFetch {}", url, md, nextFetch);
        ScheduledURL tuple = new ScheduledURL(url, md, nextFetch);
        synchronized (queue) {
            queue.add(tuple);
        }
    }

public
static
void
add
(
String
url
,
Metadata
md
,
Date
nextFetch
)
{
LOG
.
debug
(
"Adding {} with md {} and nextFetch {}"
,
url
,
md
,
nextFetch
)
;
ScheduledURL
tuple
=
new
ScheduledURL
(
url
,
md
,
nextFetch
)
;
synchronized
(
queue
)
{
queue
.
add
(
tuple
)
;
}
}


public static String cleanFieldName(String name) {
        String lowercase = name.toLowerCase();
        lowercase = lowercase.replaceAll("[^a-z_0-9]", "_");
        if (lowercase.length() < 3 || lowercase.length() > 64)
            throw new RuntimeException(
                    "Field name must be between 3 and 64 chars : " + lowercase);
        if (lowercase.equals("score"))
            throw new RuntimeException("Field name must be score");
        return lowercase;
    }

Remove the non-cloudSearch-legal characters. Note that this might convert
two fields to the same name.

@see
configuring-index-fields.html
@param name
@return

Remove the non - cloudSearch - legal characters . Note that this might convert two fields to the same name .


private static String getCharsetFromBOM(final byte[] byteData) {
        BOMInputStream bomIn = new BOMInputStream(new ByteArrayInputStream(
                byteData));
        try {
            ByteOrderMark bom = bomIn.getBOM();
            if (bom != null) {
                return bom.getCharsetName();
            }
        } catch (IOException e) {
            return null;
        }
        return null;
    }

private static String getCharsetFromBOM ( final byte [ ] byteData ) { BOMInputStream bomIn = new BOMInputStream ( new ByteArrayInputStream ( byteData ) ) ; try { ByteOrderMark bom = bomIn . getBOM ( ) ; if ( bom != null ) { return bom . getCharsetName ( ) ; } } catch ( IOException e ) { return null ; } return null ; }

Detects any BOMs and returns the corresponding charset


private static String getCharsetFromText(byte[] content,
            String declaredCharset, int maxLengthCharsetDetection) {
        String charset = null;
        // filter HTML tags
        CharsetDetector charsetDetector = new CharsetDetector();
        charsetDetector.enableInputFilter(true);
        // give it a hint
        if (declaredCharset != null)
            charsetDetector.setDeclaredEncoding(declaredCharset);
        // trim the content of the text for the detection
        byte[] subContent = content;
        if (maxLengthCharsetDetection != -1
                && content.length > maxLengthCharsetDetection) {
            subContent = Arrays.copyOfRange(content, 0,
                    maxLengthCharsetDetection);
        }
        charsetDetector.setText(subContent);
        try {
            CharsetMatch charsetMatch = charsetDetector.detect();
            charset = validateCharset(charsetMatch.getName());
        } catch (Exception e) {
            charset = null;
        }
        return charset;
    }

Use a third party library as last resort to guess the charset from the
bytes.

Use a third party library as last resort to guess the charset from the bytes .


private static String getCharsetFromMeta(byte buffer[], int maxlength) {
        // convert to UTF-8 String -- which hopefully will not mess up the
        // characters we're interested in...
        int len = buffer.length;
        if (maxlength > 0 && maxlength < len) {
            len = maxlength;
        }
        String html = new String(buffer, 0, len, DEFAULT_CHARSET);

        Document doc = Parser.htmlParser().parseInput(html, "dummy");

        // look for <meta http-equiv="Content-Type"
        // content="text/html;charset=gb2312"> or HTML5 <meta charset="gb2312">
        Elements metaElements = doc
                .select("meta[http-equiv=content-type], meta[charset]");
        String foundCharset = null;
        for (Element meta : metaElements) {
            if (meta.hasAttr("http-equiv"))
                foundCharset = getCharsetFromContentType(meta.attr("content"));
            if (foundCharset == null && meta.hasAttr("charset"))
                foundCharset = meta.attr("charset");
            if (foundCharset != null)
                return foundCharset;
        }

        return foundCharset;
    }

Attempt to find a META tag in the HTML that hints at the character set
used to write the document.

Attempt to find a META tag in the HTML that hints at the character set used to write the document .


private final boolean sniff(byte[] content) {
        byte[] beginning = content;
        if (content.length > maxOffsetGuess && maxOffsetGuess > 0) {
            beginning = Arrays.copyOfRange(content, 0, maxOffsetGuess);
        }
        int position = Bytes.indexOf(beginning, clue);
        if (position != -1) {
            return true;
        }
        return false;
    }

private final boolean sniff ( byte [ ] content ) { byte [ ] beginning = content ; if ( content . length > maxOffsetGuess && maxOffsetGuess > 0 ) { beginning = Arrays . copyOfRange ( content , 0 , maxOffsetGuess ) ; } int position = Bytes . indexOf ( beginning , clue ) ; if ( position != - 1 ) { return true ; } return false ; }

Examines the first bytes of the content for a clue of whether this
document is a sitemap, based on namespaces. Works for XML and
non-compressed documents only.

Examines the first bytes of the content for a clue of whether this document is a sitemap based on namespaces . Works for XML and non - compressed documents only .


public void setValue(String key, String value) {
        md.put(key, new String[] { value });
    }

public void setValue ( String key , String value ) { md . put ( key , new String [ ] { value } ) ; }

Set the value for a given key. The value can be null.

Set the value for a given key . The value can be null .