ReflectionTypeLoadException when load Roslyn generated assemblies

I use the following code to generate assemblys and copy all reference DLLs to the destination folder.

public HashSet<string> CompileToDll(string outputDir )
{
    var projectGraph = _sln.GetProjectDependencyGraph();
    var entries = new HashSet<string>();
    foreach (var projectID in projectGraph.GetTopologicallySortedProjects())
    {
        Compilation? projCompilation = _sln?.GetProject(projectID)?.GetCompilationAsync()?.Result;
        if (projCompilation is not null && !string.IsNullOrEmpty(projCompilation.AssemblyName))
        {
            using (var stream = new MemoryStream())
            {
                var result = projCompilation.Emit(stream);
                if (result.Success)
                {
                    string filename = $"{projCompilation.AssemblyName}.dll";
                    entries.Add(filename);
                    using (var file = File.Create($"{outputDir}\\{filename}"))
                    {
                        stream.Seek(0, SeekOrigin.Begin);
                        stream.CopyTo(file);
                    }

                    var asm = Assembly.LoadFrom($"{outputDir}\\{filename}");
                    var refs = asm.GetReferencedAssemblies();
                    CopyAllReferencesToOutput(projCompilation.References, outputDir);
                }
            }
        }
    }
    return entries;
}

Then I want to use reflection to load the DLLs to do some analysis. But when I use the following code to load assembly and get DefinedTypes, an exception was thrown.

foreach (var path in dllPaths)
{
    if (!entries.Contains(Path.GetFileName(path)))
        continue;
    var asm = Assembly.LoadFrom(path);
    var asmDefinitions = asm.DefinedTypes;
    // ...
}

Unhandled exception. System.Reflection.ReflectionTypeLoadException: Unable to load one or more of the requested types.
Could not load file or assembly 'Microsoft.AspNetCore.Mvc.Core, Version=5.0.0.0, Culture=neutral, PublicKeyToken=adb9793829ddae60'. Reference assemblies should not be loaded for execution.  They can only be loaded in the Reflection-only loader context. (0x80131058)
Could not load file or assembly 'Microsoft.AspNetCore.Mvc.ViewFeatures, Version=5.0.0.0, Culture=neutral, PublicKeyToken=adb9793829ddae60'. Reference assemblies should not be loaded for execution.  They can only be loaded in the Reflection-only loader context. (0x80131058)
Could not load file or assembly 'Microsoft.AspNetCore.Http.Abstractions, Version=5.0.0.0, Culture=neutral, PublicKeyToken=adb9793829ddae60'. Reference assemblies should not be loaded for execution.  They can only be loaded in the Reflection-only loader context. (0x80131058)
   at System.Reflection.RuntimeModule.GetTypes(RuntimeModule module)
   at System.Reflection.RuntimeAssembly.get_DefinedTypes()
   at DeserializationChecker.Program.PostProcessReflections(HashSet`1 entries, String input) in C:\workspace\DeserializationChecker\DeserializationChecker\Program.cs:line 43
   at DeserializationChecker.Program.Main(String[] args) in C:\workspace\DeserializationChecker\DeserializationChecker\Program.cs:line 29
System.BadImageFormatException: Could not load file or assembly 'Microsoft.AspNetCore.Mvc.Core, Version=5.0.0.0, Culture=neutral, PublicKeyToken=adb9793829ddae60'. Reference assemblies should not be loaded for execution.  They can only be loaded in the Reflection-only loader context. (0x80131058)
File name: 'Microsoft.AspNetCore.Mvc.Core, Version=5.0.0.0, Culture=neutral, PublicKeyToken=adb9793829ddae60' ---> System.BadImageFormatException: Could not load file or assembly 'Microsoft.AspNetCore.Mvc.Core, Version=5.0.0.0, Culture=neutral, PublicKeyToken=adb9793829ddae60'. Reference assemblies should not be loaded for execution.  They can only be loaded in the Reflection-only loader context. (0x80131058)
File name: 'Microsoft.AspNetCore.Mvc.Core, Version=5.0.0.0, Culture=neutral, PublicKeyToken=adb9793829ddae60' ---> System.BadImageFormatException: Cannot load a reference assembly for execution.
   at System.Runtime.Loader.AssemblyLoadContext.LoadFromPath(IntPtr ptrNativeAssemblyLoadContext, String ilPath, String niPath, ObjectHandleOnStack retAssembly)
   at System.Runtime.Loader.AssemblyLoadContext.LoadFromAssemblyPath(String assemblyPath)
   at System.Reflection.Assembly.LoadFrom(String assemblyFile)
   at System.Reflection.Assembly.LoadFromResolveHandler(Object sender, ResolveEventArgs args)
   at System.Runtime.Loader.AssemblyLoadContext.InvokeResolveEvent(ResolveEventHandler eventHandler, RuntimeAssembly assembly, String name)
   at System.Runtime.Loader.AssemblyLoadContext.OnAssemblyResolve(RuntimeAssembly assembly, String assemblyFullName)
System.BadImageFormatException: Could not load file or assembly 'Microsoft.AspNetCore.Mvc.ViewFeatures, Version=5.0.0.0, Culture=neutral, PublicKeyToken=adb9793829ddae60'. Reference assemblies should not be loaded for execution.  They can only be loaded in the Reflection-only loader context. (0x80131058)
File name: 'Microsoft.AspNetCore.Mvc.ViewFeatures, Version=5.0.0.0, Culture=neutral, PublicKeyToken=adb9793829ddae60' ---> System.BadImageFormatException: Could not load file or assembly 'Microsoft.AspNetCore.Mvc.ViewFeatures, Version=5.0.0.0, Culture=neutral, PublicKeyToken=adb9793829ddae60'. Reference assemblies should not be loaded for execution.  They can only be loaded in the Reflection-only loader context. (0x80131058)
File name: 'Microsoft.AspNetCore.Mvc.ViewFeatures, Version=5.0.0.0, Culture=neutral, PublicKeyToken=adb9793829ddae60' ---> System.BadImageFormatException: Cannot load a reference assembly for execution.
   at System.Runtime.Loader.AssemblyLoadContext.LoadFromPath(IntPtr ptrNativeAssemblyLoadContext, String ilPath, String niPath, ObjectHandleOnStack retAssembly)
   at System.Runtime.Loader.AssemblyLoadContext.LoadFromAssemblyPath(String assemblyPath)
   at System.Reflection.Assembly.LoadFrom(String assemblyFile)
   at System.Reflection.Assembly.LoadFromResolveHandler(Object sender, ResolveEventArgs args)
   at System.Runtime.Loader.AssemblyLoadContext.InvokeResolveEvent(ResolveEventHandler eventHandler, RuntimeAssembly assembly, String name)
   at System.Runtime.Loader.AssemblyLoadContext.OnAssemblyResolve(RuntimeAssembly assembly, String assemblyFullName)
System.BadImageFormatException: Could not load file or assembly 'Microsoft.AspNetCore.Http.Abstractions, Version=5.0.0.0, Culture=neutral, PublicKeyToken=adb9793829ddae60'. Reference assemblies should not be loaded for execution.  They can only be loaded in the Reflection-only loader context. (0x80131058)
File name: 'Microsoft.AspNetCore.Http.Abstractions, Version=5.0.0.0, Culture=neutral, PublicKeyToken=adb9793829ddae60' ---> System.BadImageFormatException: Could not load file or assembly 'Microsoft.AspNetCore.Http.Abstractions, Version=5.0.0.0, Culture=neutral, PublicKeyToken=adb9793829ddae60'. Reference assemblies should not be loaded for execution.  They can only be loaded in the Reflection-only loader context. (0x80131058)
File name: 'Microsoft.AspNetCore.Http.Abstractions, Version=5.0.0.0, Culture=neutral, PublicKeyToken=adb9793829ddae60' ---> System.BadImageFormatException: Cannot load a reference assembly for execution.
   at System.Runtime.Loader.AssemblyLoadContext.LoadFromPath(IntPtr ptrNativeAssemblyLoadContext, String ilPath, String niPath, ObjectHandleOnStack retAssembly)
   at System.Runtime.Loader.AssemblyLoadContext.LoadFromAssemblyPath(String assemblyPath)
   at System.Reflection.Assembly.LoadFrom(String assemblyFile)
   at System.Reflection.Assembly.LoadFromResolveHandler(Object sender, ResolveEventArgs args)
   at System.Runtime.Loader.AssemblyLoadContext.InvokeResolveEvent(ResolveEventHandler eventHandler, RuntimeAssembly assembly, String name)
   at System.Runtime.Loader.AssemblyLoadContext.OnAssemblyResolve(RuntimeAssembly assembly, String assemblyFullName)

But the DLL is actually in the target folder...