001 /*
002 * JBoss, Home of Professional Open Source.
003 * Copyright 2008, Red Hat Middleware LLC, and individual contributors
004 * as indicated by the @author tags. See the copyright.txt file in the
005 * distribution for a full listing of individual contributors.
006 *
007 * This is free software; you can redistribute it and/or modify it
008 * under the terms of the GNU Lesser General Public License as
009 * published by the Free Software Foundation; either version 2.1 of
010 * the License, or (at your option) any later version.
011 *
012 * This software is distributed in the hope that it will be useful,
013 * but WITHOUT ANY WARRANTY; without even the implied warranty of
014 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
015 * Lesser General Public License for more details.
016 *
017 * You should have received a copy of the GNU Lesser General Public
018 * License along with this software; if not, write to the Free
019 * Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
020 * 02110-1301 USA, or see the FSF site: http://www.fsf.org.
021 */
022 package org.jboss.dna.connector.federation.executor;
023
024 import java.util.ArrayList;
025 import java.util.Collection;
026 import java.util.Collections;
027 import java.util.HashMap;
028 import java.util.HashSet;
029 import java.util.LinkedList;
030 import java.util.List;
031 import java.util.Map;
032 import java.util.Set;
033 import java.util.UUID;
034 import java.util.concurrent.TimeUnit;
035 import net.jcip.annotations.NotThreadSafe;
036 import org.jboss.dna.common.i18n.I18n;
037 import org.jboss.dna.common.util.Logger;
038 import org.jboss.dna.connector.federation.FederationI18n;
039 import org.jboss.dna.connector.federation.Projection;
040 import org.jboss.dna.connector.federation.contribution.Contribution;
041 import org.jboss.dna.connector.federation.merge.FederatedNode;
042 import org.jboss.dna.connector.federation.merge.MergePlan;
043 import org.jboss.dna.connector.federation.merge.strategy.MergeStrategy;
044 import org.jboss.dna.connector.federation.merge.strategy.OneContributionMergeStrategy;
045 import org.jboss.dna.connector.federation.merge.strategy.SimpleMergeStrategy;
046 import org.jboss.dna.graph.DnaLexicon;
047 import org.jboss.dna.graph.ExecutionContext;
048 import org.jboss.dna.graph.cache.CachePolicy;
049 import org.jboss.dna.graph.commands.GetChildrenCommand;
050 import org.jboss.dna.graph.commands.GetNodeCommand;
051 import org.jboss.dna.graph.commands.GetPropertiesCommand;
052 import org.jboss.dna.graph.commands.GraphCommand;
053 import org.jboss.dna.graph.commands.NodeConflictBehavior;
054 import org.jboss.dna.graph.commands.basic.BasicCreateNodeCommand;
055 import org.jboss.dna.graph.commands.basic.BasicGetNodeCommand;
056 import org.jboss.dna.graph.commands.executor.AbstractCommandExecutor;
057 import org.jboss.dna.graph.connectors.RepositoryConnection;
058 import org.jboss.dna.graph.connectors.RepositoryConnectionFactory;
059 import org.jboss.dna.graph.connectors.RepositorySource;
060 import org.jboss.dna.graph.connectors.RepositorySourceException;
061 import org.jboss.dna.graph.properties.DateTime;
062 import org.jboss.dna.graph.properties.Name;
063 import org.jboss.dna.graph.properties.Path;
064 import org.jboss.dna.graph.properties.PathFactory;
065 import org.jboss.dna.graph.properties.PathNotFoundException;
066 import org.jboss.dna.graph.properties.Property;
067 import org.jboss.dna.graph.properties.Path.Segment;
068 import org.jboss.dna.graph.properties.basic.BasicSingleValueProperty;
069
070 /**
071 * @author Randall Hauch
072 */
073 @NotThreadSafe
074 public class FederatingCommandExecutor extends AbstractCommandExecutor {
075
076 private final Name uuidPropertyName;
077 private final Name mergePlanPropertyName;
078 private final CachePolicy defaultCachePolicy;
079 private final Projection cacheProjection;
080 private final List<Projection> sourceProjections;
081 private final Set<String> sourceNames;
082 private final RepositoryConnectionFactory connectionFactory;
083 private MergeStrategy mergingStrategy;
084 /** The set of all connections, including the cache connection */
085 private final Map<String, RepositoryConnection> connectionsBySourceName;
086 /** A direct reference to the cache connection */
087 private RepositoryConnection cacheConnection;
088 private Logger logger;
089
090 /**
091 * Create a command executor that federates (merges) the information from multiple sources described by the source
092 * projections. The resulting command executor does not first consult a cache for the merged information; if a cache is
093 * desired, see
094 * {@link #FederatingCommandExecutor(ExecutionContext, String, Projection, CachePolicy, List, RepositoryConnectionFactory)
095 * constructor} that takes a {@link Projection cache projection}.
096 *
097 * @param context the execution context in which the executor will be run; may not be null
098 * @param sourceName the name of the {@link RepositorySource} that is making use of this executor; may not be null or empty
099 * @param sourceProjections the source projections; may not be null
100 * @param connectionFactory the factory for {@link RepositoryConnection} instances
101 */
102 public FederatingCommandExecutor( ExecutionContext context,
103 String sourceName,
104 List<Projection> sourceProjections,
105 RepositoryConnectionFactory connectionFactory ) {
106 this(context, sourceName, null, null, sourceProjections, connectionFactory);
107 }
108
109 /**
110 * Create a command executor that federates (merges) the information from multiple sources described by the source
111 * projections. The resulting command executor will use the supplied {@link Projection cache projection} to identify the
112 * {@link Projection#getSourceName() repository source} for the cache as well as the {@link Projection#getRules() rules} for
113 * how the paths are mapped in the cache. This cache will be consulted first for the requested information, and will be kept
114 * up to date as changes are made to the federated information.
115 *
116 * @param context the execution context in which the executor will be run; may not be null
117 * @param sourceName the name of the {@link RepositorySource} that is making use of this executor; may not be null or empty
118 * @param cacheProjection the projection used for the cached information; may be null if there is no cache
119 * @param defaultCachePolicy the default caching policy that outlines the length of time that information should be cached, or
120 * null if there is no cache or no specific cache policy
121 * @param sourceProjections the source projections; may not be null
122 * @param connectionFactory the factory for {@link RepositoryConnection} instances
123 */
124 public FederatingCommandExecutor( ExecutionContext context,
125 String sourceName,
126 Projection cacheProjection,
127 CachePolicy defaultCachePolicy,
128 List<Projection> sourceProjections,
129 RepositoryConnectionFactory connectionFactory ) {
130 super(context, sourceName);
131 assert sourceProjections != null;
132 assert connectionFactory != null;
133 assert cacheProjection != null ? defaultCachePolicy != null : defaultCachePolicy == null;
134 this.cacheProjection = cacheProjection;
135 this.defaultCachePolicy = defaultCachePolicy;
136 this.sourceProjections = sourceProjections;
137 this.connectionFactory = connectionFactory;
138 this.logger = context.getLogger(getClass());
139 this.connectionsBySourceName = new HashMap<String, RepositoryConnection>();
140 this.uuidPropertyName = context.getValueFactories().getNameFactory().create(DnaLexicon.UUID);
141 this.mergePlanPropertyName = context.getValueFactories().getNameFactory().create(DnaLexicon.MERGE_PLAN);
142 this.sourceNames = new HashSet<String>();
143 for (Projection projection : this.sourceProjections) {
144 this.sourceNames.add(projection.getSourceName());
145 }
146 setMergingStrategy(null);
147 }
148
149 /**
150 * @param mergingStrategy Sets mergingStrategy to the specified value.
151 */
152 public void setMergingStrategy( MergeStrategy mergingStrategy ) {
153 if (mergingStrategy != null) {
154 this.mergingStrategy = mergingStrategy;
155 } else {
156 if (this.sourceProjections.size() == 1 && this.sourceProjections.get(0).isSimple()) {
157 this.mergingStrategy = new OneContributionMergeStrategy();
158 } else {
159 this.mergingStrategy = new SimpleMergeStrategy();
160 }
161 }
162 assert this.mergingStrategy != null;
163 }
164
165 /**
166 * Get an unmodifiable list of the immutable source projections.
167 *
168 * @return the set of projections used as sources; never null
169 */
170 public List<Projection> getSourceProjections() {
171 return Collections.unmodifiableList(sourceProjections);
172 }
173
174 /**
175 * Get the projection defining the cache.
176 *
177 * @return the cache projection
178 */
179 public Projection getCacheProjection() {
180 return cacheProjection;
181 }
182
183 /**
184 * {@inheritDoc}
185 *
186 * @see org.jboss.dna.graph.commands.executor.AbstractCommandExecutor#close()
187 */
188 @Override
189 public void close() {
190 try {
191 super.close();
192 } finally {
193 // Make sure to close ALL open connections ...
194 for (RepositoryConnection connection : connectionsBySourceName.values()) {
195 if (connection == null) continue;
196 try {
197 connection.close();
198 } catch (Throwable t) {
199 logger.debug("Error while closing connection to {0}", connection.getSourceName());
200 }
201 }
202 connectionsBySourceName.clear();
203 try {
204 if (this.cacheConnection != null) this.cacheConnection.close();
205 } finally {
206 this.cacheConnection = null;
207 }
208 }
209 }
210
211 protected RepositoryConnection getConnectionToCache() throws RepositorySourceException {
212 if (this.cacheConnection == null) {
213 this.cacheConnection = getConnection(this.cacheProjection);
214 }
215 assert this.cacheConnection != null;
216 return this.cacheConnection;
217 }
218
219 protected RepositoryConnection getConnection( Projection projection ) throws RepositorySourceException {
220 String sourceName = projection.getSourceName();
221 RepositoryConnection connection = connectionsBySourceName.get(sourceName);
222 if (connection == null) {
223 connection = connectionFactory.createConnection(sourceName);
224 connectionsBySourceName.put(sourceName, connection);
225 }
226 return connection;
227 }
228
229 protected Set<String> getOpenConnections() {
230 return connectionsBySourceName.keySet();
231 }
232
233 /**
234 * {@inheritDoc}
235 * <p>
236 * This class overrides the {@link AbstractCommandExecutor#execute(GetNodeCommand) default behavior} and instead processes the
237 * command in a more efficient manner.
238 * </p>
239 *
240 * @see org.jboss.dna.graph.commands.executor.AbstractCommandExecutor#execute(org.jboss.dna.graph.commands.GetNodeCommand)
241 */
242 @Override
243 public void execute( GetNodeCommand command ) throws RepositorySourceException {
244 BasicGetNodeCommand nodeInfo = getNode(command.getPath());
245 if (nodeInfo.hasError()) return;
246 for (Property property : nodeInfo.getProperties()) {
247 command.setProperty(property);
248 }
249 for (Segment child : nodeInfo.getChildren()) {
250 command.addChild(child, nodeInfo.getChildIdentityProperties(child));
251 }
252 }
253
254 /**
255 * {@inheritDoc}
256 *
257 * @see org.jboss.dna.graph.commands.executor.AbstractCommandExecutor#execute(org.jboss.dna.graph.commands.GetPropertiesCommand)
258 */
259 @Override
260 public void execute( GetPropertiesCommand command ) throws RepositorySourceException {
261 BasicGetNodeCommand nodeInfo = getNode(command.getPath());
262 if (nodeInfo.hasError()) return;
263 for (Property property : nodeInfo.getProperties()) {
264 command.setProperty(property);
265 }
266 }
267
268 /**
269 * {@inheritDoc}
270 *
271 * @see org.jboss.dna.graph.commands.executor.AbstractCommandExecutor#execute(org.jboss.dna.graph.commands.GetChildrenCommand)
272 */
273 @Override
274 public void execute( GetChildrenCommand command ) throws RepositorySourceException {
275 BasicGetNodeCommand nodeInfo = getNode(command.getPath());
276 if (nodeInfo.hasError()) return;
277 for (Segment child : nodeInfo.getChildren()) {
278 command.addChild(child, nodeInfo.getChildIdentityProperties(child));
279 }
280 }
281
282 /**
283 * Get the node information from the underlying sources or, if possible, from the cache.
284 *
285 * @param path the path of the node to be returned
286 * @return the node information
287 * @throws RepositorySourceException
288 */
289 protected BasicGetNodeCommand getNode( Path path ) throws RepositorySourceException {
290 // Check the cache first ...
291 final ExecutionContext context = getExecutionContext();
292 RepositoryConnection cacheConnection = getConnectionToCache();
293 BasicGetNodeCommand fromCache = new BasicGetNodeCommand(path);
294 cacheConnection.execute(context, fromCache);
295
296 // Look at the cache results from the cache for problems, or if found a plan in the cache look
297 // at the contributions. We'll be putting together the set of source names for which we need to
298 // get the contributions.
299 Set<String> sourceNames = null;
300 List<Contribution> contributions = new LinkedList<Contribution>();
301
302 if (fromCache.hasError()) {
303 Throwable error = fromCache.getError();
304 if (!(error instanceof PathNotFoundException)) return fromCache;
305
306 // The path was not found in the cache, so since we don't know whether the ancestors are federated
307 // from multiple source nodes, we need to populate the cache starting with the lowest ancestor
308 // that already exists in the cache.
309 PathNotFoundException notFound = (PathNotFoundException)fromCache.getError();
310 Path lowestExistingAncestor = notFound.getLowestAncestorThatDoesExist();
311 Path ancestor = path.getParent();
312
313 if (!ancestor.equals(lowestExistingAncestor)) {
314 // Load the nodes along the path below the existing ancestor, down to (but excluding) the desired path
315 Path pathToLoad = path.getParent();
316 while (!pathToLoad.equals(lowestExistingAncestor)) {
317 loadContributionsFromSources(pathToLoad, null, contributions); // sourceNames may be null or empty
318 FederatedNode mergedNode = createFederatedNode(null, pathToLoad, contributions, true);
319 if (mergedNode == null) {
320 // No source had a contribution ...
321 I18n msg = FederationI18n.nodeDoesNotExistAtPath;
322 fromCache.setError(new PathNotFoundException(path, ancestor, msg.text(path, ancestor)));
323 return fromCache;
324 }
325 contributions.clear();
326 // Move to the next child along the path ...
327 pathToLoad = pathToLoad.getParent();
328 }
329 }
330 // At this point, all ancestors exist ...
331 } else {
332 // There is no error, so look for the merge plan ...
333 MergePlan mergePlan = getMergePlan(fromCache);
334 if (mergePlan != null) {
335 // We found the merge plan, so check whether it's still valid ...
336 final DateTime now = getCurrentTimeInUtc();
337 if (mergePlan.isExpired(now)) {
338 // It is still valid, so check whether any contribution is from a non-existant projection ...
339 for (Contribution contribution : mergePlan) {
340 if (!this.sourceNames.contains(contribution.getSourceName())) {
341 // TODO: Record that the cached contribution is from a source that is no longer in this repository
342 }
343 }
344 return fromCache;
345 }
346
347 // At least one of the contributions is expired, so go through the contributions and place
348 // the valid contributions in the 'contributions' list; any expired contribution
349 // needs to be loaded by adding the name to the 'sourceNames'
350 if (mergePlan.getContributionCount() > 0) {
351 sourceNames = new HashSet<String>(sourceNames);
352 for (Contribution contribution : mergePlan) {
353 if (!contribution.isExpired(now)) {
354 sourceNames.remove(contribution.getSourceName());
355 contributions.add(contribution);
356 }
357 }
358 }
359 }
360 }
361
362 // Get the contributions from the sources given their names ...
363 loadContributionsFromSources(path, sourceNames, contributions); // sourceNames may be null or empty
364 FederatedNode mergedNode = createFederatedNode(fromCache, path, contributions, true);
365 if (mergedNode == null) {
366 // No source had a contribution ...
367 Path ancestor = path.getParent();
368 I18n msg = FederationI18n.nodeDoesNotExistAtPath;
369 fromCache.setError(new PathNotFoundException(path, ancestor, msg.text(path, ancestor)));
370 return fromCache;
371 }
372 return mergedNode;
373 }
374
375 protected FederatedNode createFederatedNode( BasicGetNodeCommand fromCache,
376 Path path,
377 List<Contribution> contributions,
378 boolean updateCache ) throws RepositorySourceException {
379
380 // If there are no contributions from any source ...
381 boolean foundNonEmptyContribution = false;
382 for (Contribution contribution : contributions) {
383 assert contribution != null;
384 if (!contribution.isEmpty()) {
385 foundNonEmptyContribution = true;
386 break;
387 }
388 }
389 if (!foundNonEmptyContribution) return null;
390 if (logger.isTraceEnabled()) {
391 logger.trace("Loaded {0} from sources, resulting in these contributions:", path);
392 int i = 0;
393 for (Contribution contribution : contributions) {
394 logger.trace(" {0} {1}", ++i, contribution);
395 }
396 }
397
398 // Create the node, and use the existing UUID if one is found in the cache ...
399 ExecutionContext context = getExecutionContext();
400 assert context != null;
401 UUID uuid = null;
402 if (fromCache != null) {
403 Property uuidProperty = fromCache.getPropertiesByName().get(DnaLexicon.UUID);
404 if (uuidProperty != null && !uuidProperty.isEmpty()) {
405 uuid = context.getValueFactories().getUuidFactory().create(uuidProperty.getValues().next());
406 }
407 }
408 if (uuid == null) uuid = UUID.randomUUID();
409 FederatedNode mergedNode = new FederatedNode(path, uuid);
410
411 // Merge the results into a single set of results ...
412 assert contributions.size() > 0;
413 mergingStrategy.merge(mergedNode, contributions, context);
414 if (mergedNode.getCachePolicy() == null) {
415 mergedNode.setCachePolicy(defaultCachePolicy);
416 }
417 if (updateCache) {
418 // Place the results into the cache ...
419 updateCache(mergedNode);
420 }
421 // And return the results ...
422 return mergedNode;
423 }
424
425 /**
426 * Load the node at the supplied path from the sources with the supplied name, returning the information. This method always
427 * obtains the information from the sources and does not use or update the cache.
428 *
429 * @param path the path of the node that is to be loaded
430 * @param sourceNames the names of the sources from which contributions are to be loaded; may be empty or null if all
431 * contributions from all sources are to be loaded
432 * @param contributions the list into which the contributions are to be placed
433 * @throws RepositorySourceException
434 */
435 protected void loadContributionsFromSources( Path path,
436 Set<String> sourceNames,
437 List<Contribution> contributions ) throws RepositorySourceException {
438 // At this point, there is no merge plan, so read information from the sources ...
439 ExecutionContext context = getExecutionContext();
440 PathFactory pathFactory = context.getValueFactories().getPathFactory();
441 for (Projection projection : this.sourceProjections) {
442 final String source = projection.getSourceName();
443 if (sourceNames != null && !sourceNames.contains(source)) continue;
444 final RepositoryConnection sourceConnection = getConnection(projection);
445 if (sourceConnection == null) continue; // No source exists by this name
446 // Get the cached information ...
447 CachePolicy cachePolicy = sourceConnection.getDefaultCachePolicy();
448 if (cachePolicy == null) cachePolicy = this.defaultCachePolicy;
449 DateTime expirationTime = null;
450 if (cachePolicy != null) {
451 expirationTime = getCurrentTimeInUtc().plus(cachePolicy.getTimeToLive(), TimeUnit.MILLISECONDS);
452 }
453 // Get the paths-in-source where we should fetch node contributions ...
454 Set<Path> pathsInSource = projection.getPathsInSource(path, pathFactory);
455 if (pathsInSource.isEmpty()) {
456 // The source has no contributions, but see whether the project exists BELOW this path.
457 // We do this by getting the top-level repository paths of the projection, and then
458 // use those to figure out the children of the nodes.
459 Contribution contribution = null;
460 List<Path> topLevelPaths = projection.getTopLevelPathsInRepository(pathFactory);
461 switch (topLevelPaths.size()) {
462 case 0:
463 break;
464 case 1: {
465 Path topLevelPath = topLevelPaths.iterator().next();
466 if (path.isAncestorOf(topLevelPath)) {
467 assert topLevelPath.size() > path.size();
468 Path.Segment child = topLevelPath.getSegment(path.size());
469 contribution = Contribution.createPlaceholder(source, path, expirationTime, child);
470 }
471 break;
472 }
473 default: {
474 // We assume that the top-level paths do not overlap ...
475 List<Path.Segment> children = new ArrayList<Path.Segment>(topLevelPaths.size());
476 for (Path topLevelPath : topLevelPaths) {
477 if (path.isAncestorOf(topLevelPath)) {
478 assert topLevelPath.size() > path.size();
479 Path.Segment child = topLevelPath.getSegment(path.size());
480 children.add(child);
481 }
482 }
483 if (children.size() > 0) {
484 contribution = Contribution.createPlaceholder(source, path, expirationTime, children);
485 }
486 }
487 }
488 if (contribution == null) contribution = Contribution.create(source, expirationTime);
489 contributions.add(contribution);
490 } else {
491 // There is at least one (real) contribution ...
492
493 // Get the contributions ...
494 final int numPaths = pathsInSource.size();
495 if (numPaths == 1) {
496 Path pathInSource = pathsInSource.iterator().next();
497 BasicGetNodeCommand fromSource = new BasicGetNodeCommand(pathInSource);
498 sourceConnection.execute(getExecutionContext(), fromSource);
499 if (!fromSource.hasError()) {
500 Collection<Property> properties = fromSource.getProperties();
501 List<Segment> children = fromSource.getChildren();
502 DateTime expTime = fromSource.getCachePolicy() == null ? expirationTime : getCurrentTimeInUtc().plus(fromSource.getCachePolicy().getTimeToLive(),
503 TimeUnit.MILLISECONDS);
504 Contribution contribution = Contribution.create(source, pathInSource, expTime, properties, children);
505 contributions.add(contribution);
506 }
507 } else {
508 BasicGetNodeCommand[] fromSourceCommands = new BasicGetNodeCommand[numPaths];
509 int i = 0;
510 for (Path pathInSource : pathsInSource) {
511 fromSourceCommands[i++] = new BasicGetNodeCommand(pathInSource);
512 }
513 sourceConnection.execute(context, fromSourceCommands);
514 for (BasicGetNodeCommand fromSource : fromSourceCommands) {
515 if (fromSource.hasError()) continue;
516 Collection<Property> properties = fromSource.getProperties();
517 List<Segment> children = fromSource.getChildren();
518 DateTime expTime = fromSource.getCachePolicy() == null ? expirationTime : getCurrentTimeInUtc().plus(fromSource.getCachePolicy().getTimeToLive(),
519 TimeUnit.MILLISECONDS);
520 Contribution contribution = Contribution.create(source,
521 fromSource.getPath(),
522 expTime,
523 properties,
524 children);
525 contributions.add(contribution);
526 }
527 }
528 }
529 }
530 }
531
532 protected MergePlan getMergePlan( BasicGetNodeCommand command ) {
533 Property mergePlanProperty = command.getPropertiesByName().get(mergePlanPropertyName);
534 if (mergePlanProperty == null || mergePlanProperty.isEmpty()) {
535 return null;
536 }
537 Object value = mergePlanProperty.getValues().next();
538 return value instanceof MergePlan ? (MergePlan)value : null;
539 }
540
541 protected void updateCache( FederatedNode mergedNode ) throws RepositorySourceException {
542 final ExecutionContext context = getExecutionContext();
543 final RepositoryConnection cacheConnection = getConnectionToCache();
544 final Path path = mergedNode.getPath();
545
546 NodeConflictBehavior conflictBehavior = NodeConflictBehavior.UPDATE;
547 Collection<Property> properties = new ArrayList<Property>(mergedNode.getPropertiesByName().size() + 1);
548 properties.add(new BasicSingleValueProperty(this.uuidPropertyName, mergedNode.getUuid()));
549 BasicCreateNodeCommand newNode = new BasicCreateNodeCommand(path, properties, conflictBehavior);
550 List<Segment> children = mergedNode.getChildren();
551 GraphCommand[] intoCache = new GraphCommand[1 + children.size()];
552 int i = 0;
553 intoCache[i++] = newNode;
554 List<Property> noProperties = Collections.emptyList();
555 PathFactory pathFactory = context.getValueFactories().getPathFactory();
556 for (Segment child : mergedNode.getChildren()) {
557 newNode = new BasicCreateNodeCommand(pathFactory.create(path, child), noProperties, conflictBehavior);
558 // newNode.setProperty(new BasicSingleValueProperty(this.uuidPropertyName, mergedNode.getUuid()));
559 intoCache[i++] = newNode;
560 }
561 cacheConnection.execute(context, intoCache);
562 }
563 }