Merge pull request #23 from extism/feat/reference-assemblies

feat: enable importing/exporting functions from referenced assemblies

.github/workflows/ci.yml

@@ -33,7 +33,10 @@ jobs:
 
       - name: Install Extism CLI
         run: |
-          go install github.com/extism/cli/extism@latest 
+          go install github.com/extism/cli/extism@latest
+          extism lib install --prefix ~/extism --version git
+          mkdir -p ./tests/Extism.Pdk.WasmTests/bin/Debug/net8.0
+          cp ~/extism/lib/libextism.so ./tests/Extism.Pdk.WasmTests/bin/Debug/net8.0
 
       - name: Setup .NET Core SDK
         uses: actions/[email protected]

.github/workflows/release.yml

@@ -27,7 +27,10 @@ jobs:
 
       - name: Install Extism CLI
         run: |
-          go install github.com/extism/cli/extism@latest 
+          go install github.com/extism/cli/extism@latest
+          extism lib install --prefix ~/extism --version git
+          mkdir -p ./tests/Extism.Pdk.WasmTests/bin/Debug/net8.0
+          cp ~/extism/lib/libextism.so ./tests/Extism.Pdk.WasmTests/bin/Debug/net8.0
 
       - name: Setup .NET Core SDK
         uses: actions/[email protected]

Extism.Pdk.sln

@@ -23,6 +23,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "KitchenSink", "samples\Kitc
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Extism.Pdk.WasmTests", "tests\Extism.Pdk.WasmTests\Extism.Pdk.WasmTests.csproj", "{F603C66E-1821-4E5F-94EF-09D5BB9A04A3}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SampleLib", "samples\SampleLib\SampleLib.csproj", "{FD3EBC89-BE62-402F-A5E4-D7298134658E}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -61,6 +63,10 @@ Global
 		{F603C66E-1821-4E5F-94EF-09D5BB9A04A3}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{F603C66E-1821-4E5F-94EF-09D5BB9A04A3}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{F603C66E-1821-4E5F-94EF-09D5BB9A04A3}.Release|Any CPU.Build.0 = Release|Any CPU
+		{FD3EBC89-BE62-402F-A5E4-D7298134658E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{FD3EBC89-BE62-402F-A5E4-D7298134658E}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{FD3EBC89-BE62-402F-A5E4-D7298134658E}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{FD3EBC89-BE62-402F-A5E4-D7298134658E}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -72,6 +78,7 @@ Global
 		{21861C9B-3C91-46BA-A4D1-5A919FF0113F} = {604F9655-E3D7-4E42-9D5D-91FBCACCD565}
 		{F045BCA5-71DC-483F-8D9F-D1416F6AB588} = {E54FB503-86BE-459F-A7B4-DF2AA9094CEE}
 		{F603C66E-1821-4E5F-94EF-09D5BB9A04A3} = {604F9655-E3D7-4E42-9D5D-91FBCACCD565}
+		{FD3EBC89-BE62-402F-A5E4-D7298134658E} = {E54FB503-86BE-459F-A7B4-DF2AA9094CEE}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {F37E205A-FB9F-4C44-B098-ACBF87CF9FF5}

Makefile

@@ -7,8 +7,11 @@ NUGET_API_KEY ?= $(shell env | grep NUGET_API_KEY)
 prepare:
 	dotnet build
 
-test: prepare
-	dotnet test
+test:
+	dotnet build ./src/Extism.Pdk.MSBuild
+	dotnet publish -c Release ./samples/KitchenSink
+	dotnet test ./tests/Extism.Pdk.MsBuild.Tests
+	dotnet test ./tests/Extism.Pdk.WasmTests
 
 clean:
 	dotnet clean

README.md

@@ -75,7 +75,6 @@ module MyPlugin
 
 open System
 open System.Runtime.InteropServices
-open System.Text.Json
 open Extism
 
 [<UnmanagedCallersOnly(EntryPoint = "greet")>]
@@ -179,39 +178,38 @@ Extism export functions simply take bytes in and bytes out. Those can be whateve
 
 C#:
 ```csharp
-public record Add(int A, int B);
+[JsonSerializable(typeof(Add))]
+[JsonSerializable(typeof(Sum))]
+public partial class SourceGenerationContext : JsonSerializerContext {}
+
+public record Add(int a, int b);
 public record Sum(int Result);
 
-[UnmanagedCallersOnly]
-public static int add()
+public static class Functions
 {
-    var inputJson = Pdk.GetInputString();
-    var options = new JsonSerializerOptions
+    [UnmanagedCallersOnly]
+    public static int add()
     {
-        PropertyNameCaseInsensitive = true
-    };
-
-    var parameters = JsonSerializer.Deserialize<Add>(inputJson, options);
-    var sum = new Sum(parameters.A + parameters.B);
-    var outputJson = JsonSerializer.Serialize(sum, options);
-    Pdk.SetOutput(outputJson);
-    return 0;
+        var inputJson = Pdk.GetInputString();
+        var parameters = JsonSerializer.Deserialize(inputJson, SourceGenerationContext.Defaul
+        var sum = new Sum(parameters.a + parameters.b);
+        var outputJson = JsonSerializer.Serialize(sum, SourceGenerationContext.Default.Sum);
+        Pdk.SetOutput(outputJson);
+        return 0;
+    }
 }
 ```
 
 F#:
 ```fsharp
-type Add = { A: int; B: int }
-type Sum = { Result: int }
-
 [<UnmanagedCallersOnly>]
 let add () =
     let inputJson = Pdk.GetInputString()
-    let options = JsonSerializerOptions(PropertyNameCaseInsensitive = true)
-    let parameters = JsonSerializer.Deserialize<Add>(inputJson, options)
-    
-    let sum = { Result = parameters.A + parameters.B }
-    let outputJson = JsonSerializer.Serialize(sum, options)
+    let jsonData = JsonDocument.Parse(inputJson).RootElement
+    let a = jsonData.GetProperty("a").GetInt32()
+    let b = jsonData.GetProperty("b").GetInt32()
+    let result = a + b
+    let outputJson = $"{{ \"Result\": {result} }}"
 
     Pdk.SetOutput(outputJson)
     0
@@ -222,6 +220,8 @@ extism call .\bin\Debug\net8.0\wasi-wasm\AppBundle\readmeapp.wasm --wasi add --i
 # => {"Result":41}
 ```
 
+**Note:** When enabling trimming, make sure you use the [source generation](https://learn.microsoft.com/en-us/dotnet/standard/serialization/system-text-json/source-generation) as reflection is disabled in that mode.
+
 ## Configs
 
 Configs are key-value pairs that can be passed in by the host when creating a plug-in. These can be useful to statically configure the plug-in with some data that exists across every function call. Here is a trivial example using Pdk.TryGetConfig:
@@ -299,6 +299,7 @@ let count () =
     Pdk.SetOutput(count.ToString())
 
     0
+```
 
 From [Extism CLI](https://github.com/extism/cli):
 ```
@@ -459,11 +460,84 @@ go run .
 # => Hello from Go!
 # => An argument to send to Go!
 ```
+
+### Referenced Assemblies
+
+Methods in referenced assemblies that are decorated with `[DllImport]` and `[UnmanagedCallersOnly]` are imported and exported respectively.
+
+**Note:** The library imports/exports are ignored if the app doesn't call at least one method from the library.
+
+For example, if we have a library that contains this class:
+```csharp
+namespace `MessagingBot.Pdk`;
+public class Events
+{
+    // This function will be imported  by all WASI apps that reference this library
+    [DllImport("env", EntryPoint = "send_message")]
+    public static extern void SendMessage(ulong offset);
+
+    // You can wrap the imports in your own functions to make them easier to use
+    public static void SendMessage(string message)
+    {
+        using var block = Pdk.Allocate(message);
+        SendMessage(block.Offset);
+    }
+
+    // This function will be exported by all WASI apps that reference this library
+    [UnmanagedCallersOnly]
+    public static extern void message_received(long offset);
+}
+```
+
+Then, we can reference the library in a WASI app and use the functions:
+
+```csharp
+using MessagingBot.Pdk;
+
+Events.SendMessage("Hello World!");
+```
+
+This is useful when you want to provide a common set of imports and exports that are specific to your use case.
+
 ### Optimize Size
-Normally, the .NET runtime is very conservative when trimming. This makes sure code doesn't break (when using reflection for example) but it also means large binary sizes. A hello world sample is about 20mb. To instruct the .NET compiler to be aggresive about trimming, you can try out these options:
+
+Normally, the .NET runtime is very conservative when trimming and includes a lot of metadata for debugging and exception purposes. This makes sure code doesn't break (when using reflection for example) but it also means large binary sizes. A hello world sample is about 20mb. To instruct the .NET compiler to be aggresive about trimming, you can try out these options:
 ```xml
+<Project Sdk="Microsoft.NET.Sdk">
+  <PropertyGroup>
+    <TargetFramework>net8.0</TargetFramework>
+    <RuntimeIdentifier>wasi-wasm</RuntimeIdentifier>
+    <OutputType>Exe</OutputType>
+    <PublishTrimmed>true</PublishTrimmed>
+    <WasmBuildNative>true</WasmBuildNative>
+    <WasmSingleFileBundle>true</WasmSingleFileBundle>
+
+    <!-- Note: TrimMode Full breaks Extism's global exception handling hook -->
+    <TrimMode>partial</TrimMode>
+    <DebuggerSupport>false</DebuggerSupport>
+    <EventSourceSupport>false</EventSourceSupport>
+    <UseSystemResourceKeys>true</UseSystemResourceKeys>
+    <NativeDebugSymbols>false</NativeDebugSymbols>
+  </PropertyGroup>
+</Project>
+```
+
+If you have imports in referenced assemblies, make sure [you mark them as roots](https://learn.microsoft.com/en-us/dotnet/core/deploying/trimming/trimming-options?pivots=dotnet-7-0#root-assemblies) so that they don't get trimmed:
+```xml
+<ItemGroup>
+    <TrimmerRootAssembly Include="SampleLib" />
+</ItemGroup>
+```
+
+And then, run:
 ```
+dotnet publish -c Release
+```
+
+Now, you'll have a significantly smaller `.wasm` file in `bin\Release\net8.0\wasi-wasm\AppBundle`.
+
+For more details, refer to [the official documentation](https://learn.microsoft.com/en-us/dotnet/core/deploying/trimming/trimming-options?pivots=dotnet-7-0#trimming-framework-library-features).
 
 ### Reach Out!
 
-Have a question or just want to drop in and say hi? [Hop on the Discord](https://extism.org/discord)!
+Have a question or just want to drop in and say hi? [Hop on the Discord](https://extism.org/discord)!

samples/KitchenSink/KitchenSink.csproj

@@ -10,10 +10,19 @@
 		<Nullable>enable</Nullable>
 		<AllowUnsafeBlocks>true</AllowUnsafeBlocks>
 		<TrimmerSingleWarn>false</TrimmerSingleWarn>
+
+		<!-- Note: TrimMode Full breaks Extism's global exception handling hook -->
+		<TrimMode>partial</TrimMode>
+		<DebuggerSupport>false</DebuggerSupport>
+		<EventSourceSupport>false</EventSourceSupport>
+		<UseSystemResourceKeys>true</UseSystemResourceKeys>
+		<NativeDebugSymbols>false</NativeDebugSymbols>
 	</PropertyGroup>
 
 	<ItemGroup>
 		<ProjectReference Include="..\..\src\Extism.Pdk\Extism.Pdk.csproj" />
+		<ProjectReference Include="..\SampleLib\SampleLib.csproj" />
+		<TrimmerRootAssembly Include="SampleLib" />
 	</ItemGroup>
 
 	<!--This is only necessary for ProjectReference, when using the nuget package this will not be necessary-->

samples/KitchenSink/Program.cs

@@ -2,7 +2,10 @@
 using System;
 using Extism;
 using System.Text.Json;
+using SampleLib;
+using System.Text.Json.Serialization;
 
+Class1.noop(); // Import Class1 from SampleLib so that it's included during compilation
 Console.WriteLine("Hello world!");
 
 namespace Functions
@@ -22,7 +25,7 @@ public static int Length()
         public static int Concat()
         {
             var json = Pdk.GetInput();
-            var payload = JsonSerializer.Deserialize<ConcatInput>(json);
+            var payload = JsonSerializer.Deserialize(json, SourceGenerationContext.Default.ConcatInput);
 
             if (payload is null)
             {
@@ -87,6 +90,9 @@ public static int Throw()
         }
     }
 
+    [JsonSerializable(typeof(ConcatInput))]
+    public partial class SourceGenerationContext : JsonSerializerContext {}
+
     public class ConcatInput
     {
         public string[] Parts { get; set; }

samples/SampleLib/Class1.cs

@@ -0,0 +1,19 @@
+using System.Runtime.InteropServices;
+
+namespace SampleLib;
+public class Class1
+{
+    [DllImport("env", EntryPoint = "samplelib_import")]
+    public static extern void samplelib_import();
+
+    [UnmanagedCallersOnly(EntryPoint = "samplelib_export")]
+    public static void samplelib_export()
+    {
+        samplelib_import();
+    }
+
+    public static void noop()
+    {
+
+    }
+}

samples/SampleLib/SampleLib.csproj

@@ -0,0 +1,13 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>net8.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\..\src\Extism.Pdk\Extism.Pdk.csproj" />
+  </ItemGroup>
+
+</Project>

src/Extism.Pdk.MSBuild/FFIGenerator.cs

@@ -15,21 +15,30 @@ public FFIGenerator(string extism, Action<string> logError)
             _extism = extism;
         }
 
-        public IEnumerable<FileEntry> GenerateGlueCode(AssemblyDefinition assembly)
+        public IEnumerable<FileEntry> GenerateGlueCode(AssemblyDefinition assembly, string directory)
         {
-            var exportedMethods = assembly.MainModule.Types
-                .SelectMany(t => t.Methods)
-                .Where(m => m.IsStatic && m.CustomAttributes.Any(a => a.AttributeType.FullName == "System.Runtime.InteropServices.UnmanagedCallersOnlyAttribute"))
+            var assemblies = assembly.MainModule.AssemblyReferences
+                .Where(r => !r.Name.StartsWith("System") && !r.Name.StartsWith("Microsoft") && r.Name != "Extism.Pdk")
+                .Select(r => AssemblyDefinition.ReadAssembly(Path.Combine(directory, r.Name + ".dll")))
+                .ToList();
+
+            assemblies.Add(assembly);
+
+            var types = assemblies.SelectMany(a => a.MainModule.Types).ToArray();
+
+            var exportedMethods = types
+                    .SelectMany(t => t.Methods)
+                    .Where(m => m.IsStatic && m.CustomAttributes.Any(a => a.AttributeType.FullName == "System.Runtime.InteropServices.UnmanagedCallersOnlyAttribute"))
                 .ToArray();
 
             // TODO: also find F# module functions
-            var importedMethods = assembly.MainModule.Types
+            var importedMethods = types
                 .SelectMany(t => t.Methods)
                 .Where(m => m.HasPInvokeInfo)
                 .ToArray();
 
             var files = GenerateImports(importedMethods, _extism);
-            files.Add(GenerateExports(assembly.Name.Name + ".dll", exportedMethods));
+            files.Add(GenerateExports(exportedMethods));
 
             return files;
         }
@@ -80,7 +89,7 @@ private List<FileEntry> GenerateImports(MethodDefinition[] importedMethods, stri
             return files;
         }
 
-        private FileEntry GenerateExports(string assemblyFileName, MethodDefinition[] exportedMethods)
+        private FileEntry GenerateExports(MethodDefinition[] exportedMethods)
         {
             var sb = new StringBuilder();
 
@@ -140,6 +149,7 @@ void extism_print_exception(MonoObject* exc)
 
                 foreach (var method in exportedMethods)
                 {
+                    var assemblyFileName = method.Module.Assembly.Name.Name + ".dll";
                     var attribute = method.CustomAttributes.First(a => a.AttributeType.Name == "UnmanagedCallersOnlyAttribute");
 
                     var exportName = attribute.Fields.FirstOrDefault(p => p.Name == "EntryPoint").Argument.Value?.ToString() ?? method.Name;
@@ -199,7 +209,7 @@ private string ToImportStatement(MethodDefinition method)
                 moduleName = "extism:host/user";
             }
 
-            var functionName = method.PInvokeInfo.EntryPoint ?? method.Name;
+            var functionName = string.IsNullOrEmpty(method.PInvokeInfo.EntryPoint) ? method.Name : method.PInvokeInfo.EntryPoint;
 
             if (!_types.ContainsKey(method.ReturnType.Name))
             {

src/Extism.Pdk.MSBuild/GenerateFFITask.cs

@@ -38,7 +38,7 @@ public override bool Execute()
 
             var generator = new FFIGenerator(File.ReadAllText(ExtismPath), (string message) => Log.LogError(message));
 
-            foreach (var file in generator.GenerateGlueCode(assembly))
+            foreach (var file in generator.GenerateGlueCode(assembly, Path.GetDirectoryName(AssemblyPath)))
             {
                 File.WriteAllText(Path.Combine(OutputPath, file.Name), file.Content);
             }