ShawnVN's Blog

The saga continues.  Soon after I reported the terrible timing-sensitive bug in the VS automation layer, I got a friendly email from someone smarter than me, and with a microsoft.com email address.  That's the kind I like..!

I've forgotten how most of this COM/OLE goo works, but this kind soul reminded me that OLE message filters (ala IMessageFilter, but not the one you're thinking of!) are the key to avoiding call-rejected errors from COM plumbing.

Code follows.  Caveat emptor -- but it works for me!

using System;
using System.Runtime.InteropServices;

/*
This class implements an OLE message filter, appropriate for use with the VS automation clients.

Sample usage:

    [STAThread]
    private static void Main()
    {
        // We avoid instantiating the RCW as "new DTE()", because the both PIAs for v7.0 and v7.1 point 
        // to the same CLSID (VS 7.0's)... 
        Type latestDTE = Type.GetTypeFromProgID("VisualStudio.DTE"); //ver-indep progid
        EnvDTE.DTE dte = Activator.CreateInstance(latestDTE) as EnvDTE.DTE;

        // Register an OLE message filter to avoid "call rejected" exceptions
        MessageFilter.Register();

        // Drive the VS automation interface
        Installer1.AddToolBoxTab(dte); // or whatever

        // Unplug the message filter
        MessageFilter.Revoke();
    }
*/

class MessageFilter : IOleMessageFilter
{
    //
    // Public API

    public static void Register()
    {
        IOleMessageFilter newfilter = new MessageFilter(); 

        IOleMessageFilter oldfilter = null; 
        CoRegisterMessageFilter(newfilter, out oldfilter);
    }

    public static void Revoke()
    {
        IOleMessageFilter oldfilter = null; 
        CoRegisterMessageFilter(null, out oldfilter);
    }

    //
    // IOleMessageFilter impl
    
    int IOleMessageFilter.HandleInComingCall(int dwCallType, System.IntPtr hTaskCaller, int dwTickCount, System.IntPtr lpInterfaceInfo) 
    {
        System.Diagnostics.Debug.WriteLine("IOleMessageFilter::HandleInComingCall");

        return 0; //SERVERCALL_ISHANDLED
    }

    int IOleMessageFilter.RetryRejectedCall(System.IntPtr hTaskCallee, int dwTickCount, int dwRejectType)
    {
        System.Diagnostics.Debug.WriteLine("IOleMessageFilter::RetryRejectedCall");

        if (dwRejectType == 2 ) //SERVERCALL_RETRYLATER
        {
            System.Diagnostics.Debug.WriteLine("Retry call later");
            return 99; //retry immediately if return >=0 & <100
        }
        return -1; //cancel call
    }

    int IOleMessageFilter.MessagePending(System.IntPtr hTaskCallee, int dwTickCount, int dwPendingType)
    {
        System.Diagnostics.Debug.WriteLine("IOleMessageFilter::MessagePending");

        return 2; //PENDINGMSG_WAITDEFPROCESS 
    }

    //
    // Implementation

    [DllImport("Ole32.dll")]
    private static extern int CoRegisterMessageFilter(IOleMessageFilter newfilter, out IOleMessageFilter oldfilter);
}

[ComImport(), Guid("00000016-0000-0000-C000-000000000046"),    
InterfaceTypeAttribute(ComInterfaceType.InterfaceIsIUnknown)]
interface IOleMessageFilter // deliberately renamed to avoid confusion w/ System.Windows.Forms.IMessageFilter
{
    [PreserveSig]
    int HandleInComingCall( 
        int dwCallType, 
        IntPtr hTaskCaller, 
        int dwTickCount, 
        IntPtr lpInterfaceInfo);

    [PreserveSig]
    int RetryRejectedCall( 
        IntPtr hTaskCallee, 
        int dwTickCount,
        int dwRejectType);

    [PreserveSig]
    int MessagePending( 
        IntPtr hTaskCallee, 
        int dwTickCount,
        int dwPendingType);
}