11

I have a custom SOLR index that I need to start from a few location, as to not include items I don't need. I thought the xml below was the proper location node for the index config, but it only crawls the first path.

Sitecore 8.1 U3

Any thoughts on crawling several different locations in one search config?

<locations hint="list:AddCrawler">
   <crawler type="Sitecore.ContentSearch.SitecoreItemCrawler, Sitecore.ContentSearch">
      <Database>web</Database>
      <Root>/sitecore/content/LocOne/Home/</Root>
   </crawler>
   <crawler type="Sitecore.ContentSearch.SitecoreItemCrawler, Sitecore.ContentSearch">
      <Database>web</Database>
      <Root>/sitecore/content/LocTwo/Home/</Root>
   </crawler>
</locations>

Full config file here.

15

The generic crawler name is good when you will only have one crawler in your index. If you have multiple crawlers then you need to have nodes with distinct names:

<locations hint="list:AddCrawler">
   <locOneCrawler type="Sitecore.ContentSearch.SitecoreItemCrawler, Sitecore.ContentSearch">
      <Database>web</Database>
      <Root>/sitecore/content/LocOne/Home/</Root>
   </locOneCrawler>
   <locTwoCrawler type="Sitecore.ContentSearch.SitecoreItemCrawler, Sitecore.ContentSearch">
      <Database>web</Database>
      <Root>/sitecore/content/LocTwo/Home/</Root>
   </locTwoCrawler>
</locations>

Further reading: https://ankitjoshi2409.wordpress.com/2017/02/12/creating-multiple-crawlers-for-custom-index-in-sitecore/

The following seem to work also:

<!-- "name" Attribute -->
<locations hint="list:AddCrawler">
    <crawler name="locOneCrawler" type="Sitecore.ContentSearch.SitecoreItemCrawler, Sitecore.ContentSearch">
        <Database>web</Database>
        <Root>/sitecore/content/LocOne/Home/</Root>
    </crawler>
    <crawler name="locTwoCrawler" type="Sitecore.ContentSearch.SitecoreItemCrawler, Sitecore.ContentSearch">
        <Database>web</Database>
        <Root>/sitecore/content/LocTwo/Home/</Root>
    </crawler>
</locations>

<!-- "CrawlerName" node -->
<locations hint="list:AddCrawler">
    <crawler type="Sitecore.ContentSearch.SitecoreItemCrawler, Sitecore.ContentSearch">
        <CrawlerName>locOneCrawler</CrawlerName>
        <Database>web</Database>
        <Root>/sitecore/content/LocOne/Home/</Root>
    </crawler>
    <crawler type="Sitecore.ContentSearch.SitecoreItemCrawler, Sitecore.ContentSearch">
        <CrawlerName>locTwoCrawler</CrawlerName>
        <Database>web</Database>
        <Root>/sitecore/content/LocTwo/Home/</Root>
    </crawler>
</locations>
| improve this answer | |
  • 1
    Yep. The root elements just need to be unique. – Mark Cassidy May 30 '17 at 19:27
5

Sitecore 8.1 has an issue with using two different locations for the same index and it is fixed just in Sitecore 8.2 Update 1. It is registred as a bug by Sitecore support. There is should be a package with #108165, but I haven't found that in the public access.

It was disscussed here not so far ago: Issue with multiple crawlers on Solr custom search index

| improve this answer | |
4

There is actually a scenario where you'll want to use a custom crawler over multiple crawlers:

If you use the indexing.getDependencies pipeline to force an item to be (re)indexed when another item is indexed, that pipeline is only executed within the context of a single crawler. That means that an item from one crawler cannot cause the item of another crawler to be re-indexed.

Here's a basic MultiRootCrawler that we use in production that gets around the above limitation:

/// <summary>
/// Handles multiple root locations, specified by a pipe delimited Root value
/// </summary>
public class MultiRootCrawler : SitecoreItemCrawler
{
    protected override bool IsAncestorOf(Item item)
    {
        using (new SecurityDisabler())
        using (new WriteCachesDisabler())
        {
            var rootItems = GetRootItems();

            return rootItems.Any(x => item.Paths.LongID.StartsWith(x.Paths.LongID, 
                StringComparison.InvariantCulture));
        }
    }

    public override void RebuildFromRoot(IProviderUpdateContext context, 
        IndexingOptions indexingOptions, CancellationToken cancellationToken)
    {

        Assert.ArgumentNotNull(context, "context");
        if (!base.ShouldStartIndexing(indexingOptions))
        {
            return;
        }

        var indexableRoots = this.GetRootItems()
            .Select(x => (SitecoreIndexableItem)x)
            .ToList();

        IDocumentBuilderOptions documentOptions = base.DocumentOptions;
        Assert.IsNotNull(documentOptions, "DocumentOptions");

        var eventEmitter = context.Index.Locator.GetInstance<IEvent>();

        foreach (var indexableRoot in indexableRoots)
        {
            eventEmitter.RaiseEvent("indexing:addingrecursive", 
                context.Index.Name, indexableRoot.UniqueId, indexableRoot.AbsolutePath);

            this.AddHierarchicalRecursive(indexableRoot, context, 
                this.index.Configuration, cancellationToken);

            eventEmitter.RaiseEvent("indexing:addedrecursive", 
                context.Index.Name, indexableRoot.UniqueId, indexableRoot.AbsolutePath);
        }
    }

    public override int GetContextIndexRanking(IIndexable indexable)
    {
        var sitecoreIndexableItem = indexable as SitecoreIndexableItem;
        if (sitecoreIndexableItem == null)
        {
            return 2147483647;
        }

        Item item = sitecoreIndexableItem;

        using (new SecurityDisabler())
        using (new SitecoreCachesDisabler())
        {
            foreach (var rootItem in GetRootItems())
            {
                if (item.Paths.LongID.StartsWith(
                    rootItem.Paths.LongID, StringComparison.InvariantCulture))
                {
                    return item.Axes.Level - rootItem.Axes.Level;
                }
            }
        }

        return 2147483647;
    }



    /*
     * RefreshFromRoot does refer to GetIndexableRoot, but (in 8.1 at least) 
       only after a scenario where it would have thrown a NullReferenceException, so 
       we can safetly assume it doesn't actually occur.
     */

    public new string Root
    {
        get { return base.Root; }
        set
        {
            if (value == null)
            {
                this.Roots = new string[0];
                base.Root = null;
                return;
            }

            this.Roots = value.Split('|');
            base.Root = this.Roots[0];
        }
    }

    public string[] Roots
    {
        get; set;
    }

    private List<Item> rootItems;
    private List<Item> GetRootItems()
    {
        if (this.rootItems == null)
        {
            var db = ContentSearchManager.Locator.GetInstance<IFactory>()
                .GetDatabase(this.Database);

            using (new SecurityDisabler())
            {
                this.rootItems = this.Roots
                    .Select(path => db.GetItem(path))
                    .Where(item => item != null)
                    .ToList();
            }
        }

        return this.rootItems;
    }
}
| improve this answer | |

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service, privacy policy and cookie policy

Not the answer you're looking for? Browse other questions tagged or ask your own question.