C#: Resolving dependencies beyond AppDomain

Note: I know there are several similar question on StackOverflow. I read all of them and none of them had a solution.

In my C# app I'm trying to implement a plugin system where the consumer of the plugins (e.g. a Windows app) is as decoupled as possible from the plugin libraries. The plugins all implement an interface called IPlugin, and that's all the consumer needs to know.

My solution was to create a method where, given the path to an assembly, it will read the assembly's exported types, check if one of them is a subclass of IPlugin, and if so - instantiate an object of that type and return it.

private static T LoadFileInternal(string fileName)
    {
        if (File.Exists(fileName))
        {
            var assembly = Assembly.LoadFrom(fileName);

            foreach (var type in assembly.GetExportedTypes())
            {
                if (!type.IsInterface
                    && !type.IsAbstract
                    && type.IsAssignableFrom(typeof(T)))
                {
                    try
                    {
                        return Activator.CreateInstance(type) as T;
                    }
                    catch (Exception e)
                    {
                        Context.Logger?.WriteError(e, "An error occured when trying to create an instance of \"{0}\".",
                            type.FullName);
                    }
                }
            }
        }

        return null;
    }

...and hilarity ensued.

The problem is that some of my assemblies implementing IPlugin have dependencies themselves. And it appears that:

1. If I use LoadFile to load the assembly, I am apparently using the current assembly's context, which means the CLR will be looking dependencies in the caller's paths and according to the caller's configuration, not the plugin's.
2. If I use LoadFrom that problem is resolved, but now IsAssignableFrom won't return the correct answer: because T is from a different context than the loaded assembly, IsAssignableFrom will always return false because it is considered a different type.
3. I tried using LoadFile and putting the entire thing inside a different AppDomain, which managed to return the correct answer - but when I extract the result from the different AppDomain (using GetData), the CLR tries to resolve the dependencies again the current app domain, and we're back to square one.

Here's the entire class, with AppDomain:

[Serializable]
public class TypedAssemblyLoader<T> : IDisposable, ISerializable where T : class
{
    #region members

    private const string FileName = "APP_DOMAIN_FILE_NAME";
    private const string InstantiatedObject = "INSTANTIATED_OBJECT";

    // TODO: this code used to instantiate all plugins in a separate appdomain. figure out if that's nessecary.
    private readonly AppDomain _appDomain;

    private bool _isDisposed;

    #endregion

    #region constructor

    public TypedAssemblyLoader(string applicationBase)
    {
        _appDomain = CreateDomain(applicationBase);
    }

    public TypedAssemblyLoader(SerializationInfo info, StreamingContext context)
    {
        _appDomain = (AppDomain)info.GetValue(nameof(_appDomain), typeof(AppDomain));
    }

    private AppDomain CreateDomain(string applicationBase)
    {
        var setupInfo = new AppDomainSetup
        {
            ApplicationBase = applicationBase,
            ShadowCopyFiles = "true",
            ApplicationName = "Extensibility Assembly Loader",
            // the configuration file path for the new AppDomain MUST be explicitly set.
            // if it is not explicitly set, it will default to the calling assembly's
            // config file path; if that happens the any assembly reference redirects specified
            // in the calling assembly's config file will be used when looking for assemblies,
            // which contradicts the idea of decoupling the calling assembly from the loaded
            // assemblies, which is EXACTLY what this class is for.
            // Example: if the calling assembly redirects all versions of Newtonsoft.Json.dll
            // to version 11.0.0, but one of the assemblies loaded by this class
            // (using LoadFileInternal) was compiled with version 8.0.0, then the loaded
            // assembly will not load correctly, because it is supposedly looking for version
            // 11.0.0 but it only has version 8.0.0 in its path.
            ConfigurationFile = ""
        };
        var domain = AppDomain.CreateDomain("Assembly Loader Domain", null, setupInfo);

        return domain;
    }

    #endregion

    #region implementations of ISerializable

    public void GetObjectData(SerializationInfo info, StreamingContext context)
    {
        info.AddValue(nameof(_appDomain), _appDomain);
    }

    #endregion

    #region assembly load methods

    /// <summary>
    /// Loads the assembly specified in fileName, if it is an instance of T. If not, returns null.
    /// </summary>
    /// <param name="fileName"></param>
    /// <returns></returns>
    public T LoadFile(string fileName)
    {
        _appDomain.SetData(FileName, fileName);
        _appDomain.SetData(InstantiatedObject, null);
        _appDomain.DoCallBack(LoadFileCallback);
        var result = _appDomain.GetData(InstantiatedObject);
        return (T)result;
    }

    private void LoadFileCallback()
    {
        var fileName = _appDomain.GetData(FileName) as string;
        T result = null;
        if (!string.IsNullOrWhiteSpace(fileName))
        {
            result = LoadFileInternal(fileName);
        }
        _appDomain.SetData(InstantiatedObject, result);
    }

    private static T LoadFileInternal(string fileName)
    {
        if (File.Exists(fileName))
        {
            var assembly = Assembly.LoadFrom(fileName);

            foreach (var type in assembly.GetExportedTypes())
            {
                if (!type.IsInterface
                    && !type.IsAbstract
                    && type.IsAssignableFrom(typeof(T)))
                {
                    try
                    {
                        return Activator.CreateInstance(type) as T;
                    }
                    catch (Exception e)
                    {
                        Context.Logger?.WriteError(e, "An error occured when trying to create an instance of \"{0}\".",
                            type.FullName);
                    }
                }
            }
        }

        return null;
    }

    #endregion

    #region IDisposable

    public void Dispose()
    {
        DisposeInternal();
        GC.SuppressFinalize(this);
    }

    private void DisposeInternal()
    {
        if (!_isDisposed)
        {
            try
            {
                if (!ReferenceEquals(_appDomain, null)) AppDomain.Unload(_appDomain);
            }
            catch
            {
            }

            _isDisposed = true;
        }
    }

    ~TypedAssemblyLoader()
    {
        DisposeInternal();
    }

    #endregion
}